temp disable locking

.github/workflows/binaries_dev.yml

@@ -38,13 +38,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -87,13 +87,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release4.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
+        uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/codeql.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/container_dev.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -56,7 +56,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -57,7 +57,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release5.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@c090f4e553673e6e505ea70d6a95362ee12adb94
+        uses: actions/dependency-review-action@11310527b429536e263dc6cc47873e608189ba21

.github/workflows/e2e.yml

@@ -24,13 +24,13 @@ jobs:
     timeout-minutes: 15
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
+      uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
 
     - name: Install dependencies
       run: |
@@ -42,43 +42,28 @@ jobs:
       run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
 
     - name: Run FIO
-      timeout-minutes: 15
+      timeout-minutes: 5
       run: |
         echo "Starting FIO at: $(date)"
         # Concurrent r/w
         echo 'Run randrw with size=16M bs=4k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
-
-        echo "Verify FIO at: $(date)"
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        
+        echo 'Run randrw with size=16M bs=128k'
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        
+        echo 'Run randrw with size=16M bs=1m'
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        
         # Verified write
         echo 'Run randwrite with size=16M bs=4k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
-
-    - name: Run FIO
-      timeout-minutes: 15
-      run: |
-        echo "Starting FIO at: $(date)"
-        # Concurrent r/w
-        echo 'Run randrw with size=16M bs=128k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
-
-        echo "Verify FIO at: $(date)"
-        # Verified write
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
+        
         echo 'Run randwrite with size=16M bs=128k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
-
-    - name: Run FIO
-      timeout-minutes: 15
-      run: |
-        echo "Starting FIO at: $(date)"
-        # Concurrent r/w
-        echo 'Run randrw with size=16M bs=1m'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
         
-        echo "Verify FIO at: $(date)"
-        # Verified write
         echo 'Run randwrite with size=16M bs=1m'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
 
     - name: Save logs
       if: always()

.github/workflows/go.yml

@@ -21,13 +21,13 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
+      uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
+      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
 
     - name: Get dependencies
       run: |

CODE_OF_CONDUCT.md

@@ -1,74 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-- Using welcoming and inclusive language
-- Being respectful of differing viewpoints and experiences
-- Gracefully accepting constructive criticism
-- Focusing on what is best for the community
-- Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-- The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-- Trolling, insulting/derogatory comments, and personal or political attacks
-- Public or private harassment
-- Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-- Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at <enteremailhere>. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

README.md

@@ -3,7 +3,7 @@
 
 [![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
 [![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
-[![Build Status](https://img.shields.io/github/actions/workflow/status/seaweedfs/seaweedfs/go.yml)](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
+[![Build Status](https://img.shields.io/github/workflow/status/chrislusf/seaweedfs/Go)](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
 [![GoDoc](https://godoc.org/github.com/seaweedfs/seaweedfs/weed?status.svg)](https://godoc.org/github.com/seaweedfs/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/seaweedfs/seaweedfs/wiki)
 [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
@@ -79,7 +79,7 @@ Table of Contents
 `docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
 
 ## Quick Start with Single Binary ##
-* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
+* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
 
 Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!

docker/seaweedfs.sql

@@ -3,10 +3,10 @@ CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret';
 GRANT ALL PRIVILEGES ON seaweedfs.* TO 'seaweedfs'@'%';
 FLUSH PRIVILEGES;
 USE seaweedfs;
-CREATE TABLE IF NOT EXISTS `filemeta` (
-    `dirhash`   BIGINT NOT NULL       COMMENT 'first 64 bits of MD5 hash value of directory field',
-    `name`      VARCHAR(766) NOT NULL COMMENT 'directory or file name',
-    `directory` TEXT NOT NULL         COMMENT 'full path to parent directory',
-    `meta`      LONGBLOB,
-    PRIMARY KEY (`dirhash`, `name`)
-) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+CREATE TABLE IF NOT EXISTS filemeta (
+    dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
+    name        VARCHAR(1000)  COMMENT 'directory or file name',
+    directory   TEXT           COMMENT 'full path to parent directory',
+    meta        LONGBLOB,
+    PRIMARY KEY (dirhash, name)
+) DEFAULT CHARSET=utf8;

go.mod

@@ -3,24 +3,24 @@ module github.com/seaweedfs/seaweedfs
 go 1.19
 
 require (
-	cloud.google.com/go v0.107.0 // indirect
-	cloud.google.com/go/pubsub v1.28.0
+	cloud.google.com/go v0.105.0 // indirect
+	cloud.google.com/go/pubsub v1.27.1
 	cloud.google.com/go/storage v1.28.1
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
-	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.44.184
+	github.com/Shopify/sarama v1.37.2
+	github.com/aws/aws-sdk-go v1.44.146
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eapache/go-resiliency v1.3.0 // indirect
-	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
@@ -31,9 +31,9 @@ require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-redsync/redsync/v4 v4.7.1
+	github.com/go-redsync/redsync/v4 v4.6.0
 	github.com/go-sql-driver/mysql v1.7.0
-	github.com/go-zookeeper/zk v1.0.3 // indirect
+	github.com/go-zookeeper/zk v1.0.2 // indirect
 	github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -57,12 +57,12 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.15.14 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/klauspost/reedsolomon v1.11.3
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.7
-	github.com/linxGnu/grocksdb v1.7.14
-	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/linxGnu/grocksdb v1.7.10
+	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
@@ -71,6 +71,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/olivere/elastic/v7 v7.0.32
+	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/peterh/liner v1.2.2
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -79,16 +80,16 @@ require (
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.9.0
+	github.com/prometheus/procfs v0.8.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/seaweedfs/goexif v2.0.0+incompatible
 	github.com/seaweedfs/raft v1.1.0
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.15.0
+	github.com/spf13/viper v1.14.0
 	github.com/stretchr/testify v1.8.1
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
@@ -102,78 +103,77 @@ require (
 	github.com/viant/ptrie v0.3.0
 	github.com/viant/toolbox v0.33.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.2 // indirect
-	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	go.etcd.io/etcd/client/v3 v3.5.6
-	go.mongodb.org/mongo-driver v1.11.1
+	go.mongodb.org/mongo-driver v1.11.0
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.28.0
-	gocloud.dev/pubsub/natspubsub v0.28.0
+	gocloud.dev v0.27.0
+	gocloud.dev/pubsub/natspubsub v0.27.0
 	gocloud.dev/pubsub/rabbitpubsub v0.27.0
-	golang.org/x/crypto v0.3.0 // indirect
-	golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
+	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
-	golang.org/x/net v0.5.0
-	golang.org/x/oauth2 v0.2.0 // indirect
-	golang.org/x/sys v0.4.0
-	golang.org/x/text v0.6.0 // indirect
-	golang.org/x/tools v0.5.0
+	golang.org/x/net v0.2.0
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
+	golang.org/x/sys v0.3.0
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/tools v0.3.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.107.0
+	google.golang.org/api v0.103.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
-	google.golang.org/grpc v1.52.0
+	google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect
+	google.golang.org/grpc v1.51.0
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
 	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.2 // indirect
+	modernc.org/libc v1.21.5 // indirect
 	modernc.org/mathutil v1.5.0 // indirect
 	modernc.org/memory v1.4.0 // indirect
 	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.20.1
+	modernc.org/sqlite v1.20.0
 	modernc.org/strutil v1.1.3
 	modernc.org/token v1.0.1 // indirect
 )
 
 require (
 	github.com/Jille/raft-grpc-transport v1.3.0
-	github.com/arangodb/go-driver v1.4.1
+	github.com/arangodb/go-driver v1.4.0
 	github.com/armon/go-metrics v0.4.1
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
+	github.com/google/flatbuffers v22.11.23+incompatible
 	github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
 	github.com/hashicorp/raft v1.3.11
 	github.com/hashicorp/raft-boltdb/v2 v2.2.2
 	github.com/rabbitmq/amqp091-go v1.5.0
-	github.com/schollz/progressbar/v3 v3.12.2
-	github.com/tikv/client-go/v2 v2.0.4
+	github.com/tikv/client-go/v2 v2.0.2
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.42.5
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1
+	golang.org/x/sync v0.1.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
 require (
-	cloud.google.com/go/compute v1.14.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.8.0 // indirect
+	cloud.google.com/go/compute v1.12.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/iam v0.7.0 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
-	github.com/aws/smithy-go v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.12.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.17.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.16.10 // indirect
+	github.com/aws/smithy-go v1.12.0 // indirect
 	github.com/benbjohnson/clock v1.1.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
@@ -182,8 +182,8 @@ require (
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fclairamb/go-log v0.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/go-hclog v1.2.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -196,43 +196,38 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
-	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/montanaflynn/stats v0.6.6 // indirect
-	github.com/nats-io/nats.go v1.20.0 // indirect
+	github.com/nats-io/nats.go v1.16.0 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/philhofer/fwd v1.1.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
-	github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect
+	github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 // indirect
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
-	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stathat/consistent v1.0.0 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
-	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
+	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 // indirect
 	github.com/tinylib/msgp v1.1.6 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba // indirect
 	github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.6 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.6 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
+	go.uber.org/zap v1.22.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/term v0.4.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.41"
-version: "3.41"
+appVersion: "3.36"
+version: "3.36"

k8s/helm_charts2/README.md

@@ -14,13 +14,13 @@ with ENV.
 A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. 
 This database should be pre-configured and initialized by running:
 ```sql
-CREATE TABLE IF NOT EXISTS `filemeta` (
-  `dirhash`   BIGINT NOT NULL       COMMENT 'first 64 bits of MD5 hash value of directory field',
-  `name`      VARCHAR(766) NOT NULL COMMENT 'directory or file name',
-  `directory` TEXT NOT NULL         COMMENT 'full path to parent directory',
-  `meta`      LONGBLOB,
-  PRIMARY KEY (`dirhash`, `name`)
-) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+CREATE TABLE IF NOT EXISTS filemeta (
+  dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
+  name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
+  directory   TEXT BINARY          COMMENT 'full path to parent directory',
+  meta        LONGBLOB,
+  PRIMARY KEY (dirhash, name)
+) DEFAULT CHARSET=utf8;
 ```
 
 Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.

k8s/helm_charts2/templates/filer-statefulset.yaml

@@ -101,12 +101,7 @@ spec:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed \
-              {{- if eq .Values.filer.logs.type "hostPath" }}
-              -logdir=/logs \
-              {{- else }}
-              -logtostderr=true \
-              {{- end }}
+              exec /usr/bin/weed -logdir=/logs \
               {{- if .Values.filer.loggingOverrideLevel }}
               -v={{ .Values.filer.loggingOverrideLevel }} \
               {{- else }}
@@ -230,13 +225,13 @@ spec:
         {{- if eq .Values.filer.logs.type "hostPath" }}
         - name: seaweedfs-filer-log-volume
           hostPath:
-            path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer
+            path: /storage/logs/seaweedfs/filer
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.filer.data.type "hostPath" }}
         - name: data-filer
           hostPath:
-            path: {{ .Values.filer.data.hostPathPrefix }}/filer_store
+            path: /storage/filer_store
             type: DirectoryOrCreate
         {{- end }}
         - name: db-schema-config-volume

k8s/helm_charts2/templates/master-statefulset.yaml

@@ -90,12 +90,7 @@ spec:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed \
-              {{- if eq .Values.master.logs.type "hostPath" }}
-              -logdir=/logs \
-              {{- else }}
-              -logtostderr=true \
-              {{- end }}
+              exec /usr/bin/weed -logdir=/logs \
               {{- if .Values.master.loggingOverrideLevel }}
               -v={{ .Values.master.loggingOverrideLevel }} \
               {{- else }}
@@ -201,13 +196,13 @@ spec:
         {{- if eq .Values.master.logs.type "hostPath" }}
         - name: seaweedfs-master-log-volume
           hostPath:
-            path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master
+            path: /storage/logs/seaweedfs/master
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.master.data.type "hostPath" }}
         - name: data-{{ .Release.Namespace }}
           hostPath:
-            path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/
+            path: /ssd/seaweed-master/
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/templates/s3-deployment.yaml

@@ -72,19 +72,13 @@ spec:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed \
-               {{- if eq .Values.s3.logs.type "hostPath" }}
-              -logdir=/logs \
-              {{- else }}
-              -logtostderr=true \
-              {{- end }}
+              exec /usr/bin/weed -logdir=/logs \
               {{- if .Values.s3.loggingOverrideLevel }}
               -v={{ .Values.s3.loggingOverrideLevel }} \
               {{- else }}
               -v={{ .Values.global.loggingLevel }} \
               {{- end }}
               s3 \
-              -ip.bind={{ .Values.s3.bindAddress }} \
               -port={{ .Values.s3.port }} \
               {{- if .Values.s3.metricsPort }}
               -metricsPort {{ .Values.s3.metricsPort }} \
@@ -173,7 +167,7 @@ spec:
         {{- if eq .Values.s3.logs.type "hostPath" }}
         - name: logs
           hostPath:
-            path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3
+            path: /storage/logs/seaweedfs/s3
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/templates/volume-statefulset.yaml

@@ -93,12 +93,7 @@ spec:
             - "/bin/sh"
             - "-ec"
             - |
-              exec /usr/bin/weed \
-                {{- if eq .Values.volume.logs.type "hostPath" }}
-                -logdir=/logs \
-                {{- else }}
-                -logtostderr=true \
-                {{- end }}
+              exec /usr/bin/weed -logdir=/logs \
                 {{- if .Values.volume.loggingOverrideLevel }}
                 -v={{ .Values.volume.loggingOverrideLevel }} \
                 {{- else }}
@@ -211,19 +206,19 @@ spec:
         {{- if eq .Values.volume.data.type "hostPath" }}
         - name: data
           hostPath:
-            path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
+            path: /storage/object_store/
             type: DirectoryOrCreate
         {{- end }}
         {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
         - name: idx
           hostPath:
-            path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
+            path: /ssd/seaweedfs-volume-idx/
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.volume.logs.type "hostPath" }}
         - name: logs
           hostPath:
-            path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
+            path: /storage/logs/seaweedfs/volume
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/values.yaml

@@ -68,13 +68,11 @@ master:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /ssd
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
 
   initContainers: ""
 
@@ -178,19 +176,15 @@ volume:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
-
   idx:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /ssd
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
 
   # limit background compaction or copying speed in mega bytes per second
   compactionMBps: "50"
@@ -314,13 +308,11 @@ filer:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
 
   initContainers: ""
 
@@ -426,7 +418,6 @@ s3:
   imageTag: null
   restartPolicy: null
   replicas: 1
-  bindAddress: 0.0.0.0
   port: 8333
   metricsPort: 9327
   loggingOverrideLevel: null
@@ -472,7 +463,7 @@ s3:
     type: "hostPath"
     size: ""
     storageClass: ""
-    hostPathPrefix: /storage
+
 
 certificates:
   commonName: "SeaweedFS CA"

unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@@ -6,7 +6,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"strconv"
@@ -53,7 +52,7 @@ func main() {
 }
 
 func startGenerateMetadata() {
-	pb.WithFilerClient(false, util.RandomInt32(), pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
 
 		for i := 0; i < *n; i++ {
 			name := fmt.Sprintf("file%d", i)

weed/command/filer.go

@@ -113,7 +113,6 @@ func init() {
 	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
 	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
-	filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
 
 	// start iam on filer
 	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")

weed/command/filer_cat.go

@@ -96,7 +96,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 		writer = f
 	}
 
-	pb.WithFilerClient(false, util.RandomInt32(), filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(false, filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Name:      name,

weed/command/filer_copy.go

@@ -159,7 +159,6 @@ func runCopy(cmd *Command, args []string) bool {
 			worker := FileCopyWorker{
 				options:      &copy,
 				filerAddress: filerAddress,
-				signature:    util.RandomInt32(),
 			}
 			if err := worker.copyFiles(fileCopyTaskChan); err != nil {
 				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
@@ -173,7 +172,7 @@ func runCopy(cmd *Command, args []string) bool {
 }
 
 func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress pb.ServerAddress) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
-	err = pb.WithGrpcFilerClient(false, 0, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
 			return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
@@ -226,7 +225,6 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 type FileCopyWorker struct {
 	options      *CopyOptions
 	filerAddress pb.ServerAddress
-	signature    int32
 }
 
 func (worker *FileCopyWorker) copyFiles(fileCopyTaskChan chan FileCopyTask) error {
@@ -304,7 +302,7 @@ func (worker *FileCopyWorker) checkExistingFileFirst(task FileCopyTask, f *os.Fi
 		return
 	}
 
-	err = pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: task.destinationUrlPath,
@@ -367,10 +365,10 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 		if flushErr != nil {
 			return flushErr
 		}
-		chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0, time.Now().UnixNano()))
+		chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0))
 	}
 
-	if err := pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -452,7 +450,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 				uploadError = fmt.Errorf("upload %v result: %v\n", fileName, uploadResult.Error)
 				return
 			}
-			chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize, time.Now().UnixNano())
+			chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize)
 
 			fmt.Printf("uploaded %s-%d [%d,%d)\n", fileName, i+1, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 		}(i)
@@ -481,7 +479,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		return fmt.Errorf("create manifest: %v", manifestErr)
 	}
 
-	if err := pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -532,7 +530,7 @@ func detectMimeType(f *os.File) string {
 	return mimeType
 }
 
-func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error) {
+func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, err error) {
 
 	finalFileId, uploadResult, flushErr, _ := operation.UploadWithRetry(
 		worker,
@@ -563,7 +561,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
 	if uploadResult.Error != "" {
 		return nil, fmt.Errorf("upload result: %v", uploadResult.Error)
 	}
-	return uploadResult.ToPbFileChunk(finalFileId, offset, tsNs), nil
+	return uploadResult.ToPbFileChunk(finalFileId, offset), nil
 }
 
 var _ = filer_pb.FilerClient(&FileCopyWorker{})
@@ -571,7 +569,7 @@ var _ = filer_pb.FilerClient(&FileCopyWorker{})
 func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
 
 	filerGrpcAddress := worker.filerAddress.ToGrpcAddress()
-	err = pb.WithGrpcClient(streamingMode, worker.signature, func(grpcConnection *grpc.ClientConn) error {
+	err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
 	}, filerGrpcAddress, false, worker.options.grpcDialOption)

weed/command/filer_meta_backup.go

@@ -225,7 +225,7 @@ var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{})
 
 func (metaBackup *FilerMetaBackupOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	return pb.WithFilerClient(streamingMode, metaBackup.clientId, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 

weed/command/filer_remote_gateway.go

@@ -35,7 +35,7 @@ type RemoteGatewayOptions struct {
 var _ = filer_pb.FilerClient(&RemoteGatewayOptions{})
 
 func (option *RemoteGatewayOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
-	return pb.WithFilerClient(streamingMode, option.clientId, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 }

weed/command/filer_remote_sync.go

@@ -26,7 +26,7 @@ type RemoteSyncOptions struct {
 var _ = filer_pb.FilerClient(&RemoteSyncOptions{})
 
 func (option *RemoteSyncOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
-	return pb.WithFilerClient(streamingMode, option.clientId, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 }

weed/command/filer_sync.go

@@ -304,7 +304,7 @@ func getSignaturePrefixByPath(path string) string {
 
 func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
 
-	readErr = pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		syncKey := []byte(signaturePrefix + "____")
 		util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature))
 
@@ -330,7 +330,7 @@ func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature
 }
 
 func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error {
-	return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		syncKey := []byte(signaturePrefix + "____")
 		util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature))
@@ -399,11 +399,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-			if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
-				return fmt.Errorf("create entry1 : %v", err)
-			} else {
-				return nil
-			}
+			return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
 		}
 
 		// this is something special?
@@ -431,11 +427,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 				}
 				// create the new entry
 				newKey := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-				if err := dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures); err != nil {
-					return fmt.Errorf("create entry2 : %v", err)
-				} else {
-					return nil
-				}
+				return dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures)
 
 			} else {
 				// new key is outside of the watched directory
@@ -449,11 +441,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
 				// new key is in the watched directory
 				key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-				if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
-					return fmt.Errorf("create entry3 : %v", err)
-				} else {
-					return nil
-				}
+				return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
 			} else {
 				// new key is also outside of the watched directory
 				// skip

weed/command/iam.go

@@ -50,7 +50,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	for {
-		err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)

weed/command/mount.go

@@ -53,9 +53,9 @@ func init() {
 	mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
-	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers")
+	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
 	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
-	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "file chunk read cache capacity in MB")
+	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")

weed/command/mount_std.go

@@ -256,8 +256,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	seaweedFileSystem.StartBackgroundTasks()
 
-	glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
-	glog.V(0).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)
+	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
 
 	server.Serve()
 

weed/command/s3.go

@@ -163,7 +163,7 @@ func (s3opt *S3Options) startS3Server() bool {
 	var metricsIntervalSec int
 
 	for {
-		err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)

weed/command/scaffold/filer.toml

@@ -41,13 +41,13 @@ enabled = false
 dbFile = "./filer.db"                # sqlite db file
 
 [mysql]  # or memsql, tidb
-# CREATE TABLE IF NOT EXISTS `filemeta` (
-#   `dirhash`   BIGINT NOT NULL       COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   `name`      VARCHAR(766) NOT NULL COMMENT 'directory or file name',
-#   `directory` TEXT NOT NULL         COMMENT 'full path to parent directory',
-#   `meta`      LONGBLOB,
-#   PRIMARY KEY (`dirhash`, `name`)
-# ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+# CREATE TABLE IF NOT EXISTS filemeta (
+#   dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
+#   directory   TEXT BINARY          COMMENT 'full path to parent directory',
+#   meta        LONGBLOB,
+#   PRIMARY KEY (dirhash, name)
+# ) DEFAULT CHARSET=utf8;
 
 enabled = false
 hostname = "localhost"
@@ -61,18 +61,18 @@ connection_max_lifetime_seconds = 0
 interpolateParams = false
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
+upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
 
 [mysql2]  # or memsql, tidb
 enabled = false
 createTable = """
   CREATE TABLE IF NOT EXISTS `%s` (
-    `dirhash`   BIGINT NOT NULL,
-    `name`      VARCHAR(766) NOT NULL,
-    `directory` TEXT NOT NULL,
-    `meta`      LONGBLOB,
-    PRIMARY KEY (`dirhash`, `name`)
-  ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+    dirhash BIGINT,
+    name VARCHAR(1000) BINARY,
+    directory TEXT BINARY,
+    meta LONGBLOB,
+    PRIMARY KEY (dirhash, name)
+  ) DEFAULT CHARSET=utf8;
 """
 hostname = "localhost"
 port = 3306
@@ -85,7 +85,7 @@ connection_max_lifetime_seconds = 0
 interpolateParams = false
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
+upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
 
 [postgres] # or cockroachdb, YugabyteDB
 # CREATE TABLE IF NOT EXISTS filemeta (

weed/command/server.go

@@ -156,7 +156,6 @@ func init() {
 	webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
 	webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
-	webdavOptions.filerRootPath = cmdServer.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
 
 	mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port")
 

weed/command/webdav.go

@@ -13,7 +13,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
-	weed_server "github.com/seaweedfs/seaweedfs/weed/server"
+	"github.com/seaweedfs/seaweedfs/weed/server"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
@@ -23,7 +23,6 @@ var (
 
 type WebDavOption struct {
 	filer          *string
-	filerRootPath  *string
 	port           *int
 	collection     *string
 	replication    *string
@@ -45,7 +44,6 @@ func init() {
 	webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
 	webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
 	webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB")
-	webDavStandaloneOptions.filerRootPath = cmdWebDav.Flag.String("filer.path", "/", "use this remote path from filer server")
 }
 
 var cmdWebDav = &Command{
@@ -87,7 +85,7 @@ func (wo *WebDavOption) startWebDav() bool {
 	var cipher bool
 	// connect to filer
 	for {
-		err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
@@ -106,7 +104,6 @@ func (wo *WebDavOption) startWebDav() bool {
 
 	ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
 		Filer:          filerAddress,
-		FilerRootPath:  *wo.filerRootPath,
 		GrpcDialOption: grpcDialOption,
 		Collection:     *wo.collection,
 		Replication:    *wo.replication,

weed/filer/configuration.go

@@ -33,7 +33,7 @@ func (f *Filer) LoadConfiguration(config *util.ViperProxy) (isFresh bool) {
 
 	if !hasDefaultStoreConfigured {
 		println()
-		println("Supported filer stores are the following. If not found, check the full version.")
+		println("Supported filer stores are:")
 		for _, store := range Stores {
 			println("    " + store.GetName())
 		}

weed/filer/filechunk_group.go

@@ -1,155 +0,0 @@
-package filer
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
-	"github.com/seaweedfs/seaweedfs/weed/wdclient"
-	"sync"
-)
-
-type ChunkGroup struct {
-	lookupFn     wdclient.LookupFileIdFunctionType
-	chunkCache   chunk_cache.ChunkCache
-	sections     map[SectionIndex]*FileChunkSection
-	sectionsLock sync.RWMutex
-	readerCache  *ReaderCache
-}
-
-func NewChunkGroup(lookupFn wdclient.LookupFileIdFunctionType, chunkCache chunk_cache.ChunkCache, chunks []*filer_pb.FileChunk) (*ChunkGroup, error) {
-	group := &ChunkGroup{
-		lookupFn:    lookupFn,
-		chunkCache:  chunkCache,
-		sections:    make(map[SectionIndex]*FileChunkSection),
-		readerCache: NewReaderCache(32, chunkCache, lookupFn),
-	}
-
-	err := group.SetChunks(chunks)
-	return group, err
-}
-
-func (group *ChunkGroup) AddChunk(chunk *filer_pb.FileChunk) error {
-
-	group.sectionsLock.Lock()
-	defer group.sectionsLock.Unlock()
-
-	sectionIndexStart, sectionIndexStop := SectionIndex(chunk.Offset/SectionSize), SectionIndex((chunk.Offset+int64(chunk.Size))/SectionSize)
-	for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
-		section, found := group.sections[si]
-		if !found {
-			section = NewFileChunkSection(si)
-			group.sections[si] = section
-		}
-		section.addChunk(chunk)
-	}
-	return nil
-}
-
-func (group *ChunkGroup) ReadDataAt(fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
-
-	group.sectionsLock.RLock()
-	defer group.sectionsLock.RUnlock()
-
-	sectionIndexStart, sectionIndexStop := SectionIndex(offset/SectionSize), SectionIndex((offset+int64(len(buff)))/SectionSize)
-	for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
-		section, found := group.sections[si]
-		rangeStart, rangeStop := max(offset, int64(si*SectionSize)), min(offset+int64(len(buff)), int64((si+1)*SectionSize))
-		if !found {
-			for i := rangeStart; i < rangeStop; i++ {
-				buff[i-offset] = 0
-			}
-			continue
-		}
-		xn, xTsNs, xErr := section.readDataAt(group, fileSize, buff[rangeStart-offset:rangeStop-offset], rangeStart)
-		if xErr != nil {
-			err = xErr
-		}
-		n += xn
-		tsNs = max(tsNs, xTsNs)
-	}
-	return
-}
-
-func (group *ChunkGroup) SetChunks(chunks []*filer_pb.FileChunk) error {
-	group.sectionsLock.RLock()
-	defer group.sectionsLock.RUnlock()
-
-	var dataChunks []*filer_pb.FileChunk
-	for _, chunk := range chunks {
-
-		if !chunk.IsChunkManifest {
-			dataChunks = append(dataChunks, chunk)
-			continue
-		}
-
-		resolvedChunks, err := ResolveOneChunkManifest(group.lookupFn, chunk)
-		if err != nil {
-			return err
-		}
-
-		dataChunks = append(dataChunks, resolvedChunks...)
-	}
-
-	sections := make(map[SectionIndex]*FileChunkSection)
-
-	for _, chunk := range dataChunks {
-		sectionIndexStart, sectionIndexStop := SectionIndex(chunk.Offset/SectionSize), SectionIndex((chunk.Offset+int64(chunk.Size))/SectionSize)
-		for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
-			section, found := sections[si]
-			if !found {
-				section = NewFileChunkSection(si)
-				sections[si] = section
-			}
-			section.chunks = append(section.chunks, chunk)
-		}
-	}
-
-	group.sections = sections
-	return nil
-}
-
-const (
-	// see weedfs_file_lseek.go
-	SEEK_DATA uint32 = 3 // seek to next data after the offset
-	// SEEK_HOLE uint32 = 4 // seek to next hole after the offset
-)
-
-// FIXME: needa tests
-func (group *ChunkGroup) SearchChunks(offset, fileSize int64, whence uint32) (found bool, out int64) {
-	group.sectionsLock.RLock()
-	defer group.sectionsLock.RUnlock()
-
-	return group.doSearchChunks(offset, fileSize, whence)
-}
-
-func (group *ChunkGroup) doSearchChunks(offset, fileSize int64, whence uint32) (found bool, out int64) {
-
-	sectionIndex, maxSectionIndex := SectionIndex(offset/SectionSize), SectionIndex(fileSize/SectionSize)
-	if whence == SEEK_DATA {
-		for si := sectionIndex; si < maxSectionIndex+1; si++ {
-			section, foundSection := group.sections[si]
-			if !foundSection {
-				continue
-			}
-			sectionStart := section.DataStartOffset(group, offset, fileSize)
-			if sectionStart == -1 {
-				continue
-			}
-			return true, sectionStart
-		}
-		return false, 0
-	} else {
-		// whence == SEEK_HOLE
-		for si := sectionIndex; si < maxSectionIndex; si++ {
-			section, foundSection := group.sections[si]
-			if !foundSection {
-				return true, offset
-			}
-			holeStart := section.NextStopOffset(group, offset, fileSize)
-			if holeStart%SectionSize == 0 {
-				continue
-			}
-			return true, holeStart
-		}
-		return true, fileSize
-	}
-}

weed/filer/filechunk_group_test.go

@@ -1,36 +0,0 @@
-package filer
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func TestChunkGroup_doSearchChunks(t *testing.T) {
-	type fields struct {
-		sections map[SectionIndex]*FileChunkSection
-	}
-	type args struct {
-		offset   int64
-		fileSize int64
-		whence   uint32
-	}
-	tests := []struct {
-		name      string
-		fields    fields
-		args      args
-		wantFound bool
-		wantOut   int64
-	}{
-		// TODO: Add test cases.
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			group := &ChunkGroup{
-				sections: tt.fields.sections,
-			}
-			gotFound, gotOut := group.doSearchChunks(tt.args.offset, tt.args.fileSize, tt.args.whence)
-			assert.Equalf(t, tt.wantFound, gotFound, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence)
-			assert.Equalf(t, tt.wantOut, gotOut, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence)
-		})
-	}
-}

weed/filer/filechunk_manifest.go

@@ -264,7 +264,7 @@ func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer
 		}
 	}
 
-	manifestChunk, err = saveFunc(bytes.NewReader(data), "", 0, 0)
+	manifestChunk, err = saveFunc(bytes.NewReader(data), "", 0)
 	if err != nil {
 		return nil, err
 	}
@@ -275,4 +275,4 @@ func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer
 	return
 }
 
-type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error)
+type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, err error)

weed/filer/filechunk_section.go

@@ -1,134 +0,0 @@
-package filer
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"sync"
-)
-
-const SectionSize = 2 * 1024 * 1024 * 32 // 64MiB
-type SectionIndex int64
-type FileChunkSection struct {
-	sectionIndex     SectionIndex
-	chunks           []*filer_pb.FileChunk
-	visibleIntervals *IntervalList[*VisibleInterval]
-	chunkViews       *IntervalList[*ChunkView]
-	reader           *ChunkReadAt
-	lock             sync.Mutex
-}
-
-func NewFileChunkSection(si SectionIndex) *FileChunkSection {
-	return &FileChunkSection{
-		sectionIndex: si,
-	}
-}
-
-func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error {
-	section.lock.Lock()
-	defer section.lock.Unlock()
-
-	start, stop := max(int64(section.sectionIndex)*SectionSize, chunk.Offset), min(((int64(section.sectionIndex)+1)*SectionSize), chunk.Offset+int64(chunk.Size))
-
-	section.chunks = append(section.chunks, chunk)
-
-	if section.visibleIntervals == nil {
-		section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
-	} else {
-		MergeIntoVisibles(section.visibleIntervals, start, stop, chunk)
-		garbageFileIds := FindGarbageChunks(section.visibleIntervals, start, stop)
-		removeGarbageChunks(section, garbageFileIds)
-	}
-
-	if section.chunkViews != nil {
-		MergeIntoChunkViews(section.chunkViews, start, stop, chunk)
-	}
-
-	return nil
-}
-
-func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]struct{}) {
-	for i := 0; i < len(section.chunks); {
-		t := section.chunks[i]
-		length := len(section.chunks)
-		if _, found := garbageFileIds[t.FileId]; found {
-			if i < length-1 {
-				section.chunks[i] = section.chunks[length-1]
-			}
-			section.chunks = section.chunks[:length-1]
-		} else {
-			i++
-		}
-	}
-}
-
-func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64) {
-	if section.visibleIntervals == nil {
-		section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
-		section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks)
-		if section.reader != nil {
-			_ = section.reader.Close()
-			section.reader = nil
-		}
-	}
-	if section.chunkViews == nil {
-		section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
-	}
-
-	if section.reader == nil {
-		section.reader = NewChunkReaderAtFromClient(group.readerCache, section.chunkViews, min(int64(section.sectionIndex+1)*SectionSize, fileSize))
-	}
-	section.reader.fileSize = fileSize
-}
-
-func (section *FileChunkSection) readDataAt(group *ChunkGroup, fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
-	section.lock.Lock()
-	defer section.lock.Unlock()
-
-	section.setupForRead(group, fileSize)
-
-	return section.reader.ReadAtWithTime(buff, offset)
-}
-
-func (section *FileChunkSection) DataStartOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
-	section.lock.Lock()
-	defer section.lock.Unlock()
-
-	section.setupForRead(group, fileSize)
-
-	for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
-		visible := x.Value
-		if visible.stop <= offset {
-			continue
-		}
-		if offset < visible.start {
-			return offset
-		}
-		return offset
-	}
-	return -1
-}
-
-func (section *FileChunkSection) NextStopOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
-	section.lock.Lock()
-	defer section.lock.Unlock()
-
-	section.setupForRead(group, fileSize)
-
-	isAfterOffset := false
-	for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
-		visible := x.Value
-		if !isAfterOffset {
-			if visible.stop <= offset {
-				continue
-			}
-			isAfterOffset = true
-		}
-		if offset < visible.start {
-			return offset
-		}
-		// now visible.start <= offset
-		if offset < visible.stop {
-			offset = visible.stop
-		}
-	}
-	return offset
-}

weed/filer/filechunk_section_test.go

@@ -1,48 +0,0 @@
-package filer
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"testing"
-)
-
-func Test_removeGarbageChunks(t *testing.T) {
-	section := NewFileChunkSection(0)
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "0",
-		Offset:       0,
-		Size:         1,
-		ModifiedTsNs: 0,
-	})
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "1",
-		Offset:       1,
-		Size:         1,
-		ModifiedTsNs: 1,
-	})
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "2",
-		Offset:       2,
-		Size:         1,
-		ModifiedTsNs: 2,
-	})
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "3",
-		Offset:       3,
-		Size:         1,
-		ModifiedTsNs: 3,
-	})
-	section.chunks = append(section.chunks, &filer_pb.FileChunk{
-		FileId:       "4",
-		Offset:       4,
-		Size:         1,
-		ModifiedTsNs: 4,
-	})
-	garbageFileIds := make(map[string]struct{})
-	garbageFileIds["0"] = struct{}{}
-	garbageFileIds["2"] = struct{}{}
-	garbageFileIds["4"] = struct{}{}
-	removeGarbageChunks(section, garbageFileIds)
-	if len(section.chunks) != 2 {
-		t.Errorf("remove chunk 2 failed")
-	}
-}

weed/filer/filechunks.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"golang.org/x/exp/slices"
 	"math"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -41,7 +42,7 @@ func ETag(entry *filer_pb.Entry) (etag string) {
 }
 
 func ETagEntry(entry *Entry) (etag string) {
-	if entry.IsInRemoteOnly() {
+        if entry.IsInRemoteOnly() {
 		return entry.Remote.RemoteETag
 	}
 	if entry.Attr.Md5 == nil {
@@ -65,15 +66,8 @@ func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks
 
 	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, 0, math.MaxInt64)
 
-	compacted, garbage = SeparateGarbageChunks(visibles, chunks)
-
-	return
-}
-
-func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*filer_pb.FileChunk) (compacted []*filer_pb.FileChunk, garbage []*filer_pb.FileChunk) {
 	fileIds := make(map[string]bool)
-	for x := visibles.Front(); x != nil; x = x.Next {
-		interval := x.Value
+	for _, interval := range visibles {
 		fileIds[interval.fileId] = true
 	}
 	for _, chunk := range chunks {
@@ -83,18 +77,7 @@ func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*f
 			garbage = append(garbage, chunk)
 		}
 	}
-	return compacted, garbage
-}
 
-func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, stop int64) (garbageFileIds map[string]struct{}) {
-	garbageFileIds = make(map[string]struct{})
-	for x := visibles.Front(); x != nil; x = x.Next {
-		interval := x.Value
-		offset := interval.start - interval.offsetInChunk
-		if start <= offset && offset+int64(interval.chunkSize) <= stop {
-			garbageFileIds[interval.fileId] = struct{}{}
-		}
-	}
 	return
 }
 
@@ -148,39 +131,20 @@ func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_p
 }
 
 type ChunkView struct {
-	FileId        string
-	OffsetInChunk int64 // offset within the chunk
-	ViewSize      uint64
-	ViewOffset    int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
-	ChunkSize     uint64
-	CipherKey     []byte
-	IsGzipped     bool
-	ModifiedTsNs  int64
-}
-
-func (cv *ChunkView) SetStartStop(start, stop int64) {
-	cv.OffsetInChunk += start - cv.ViewOffset
-	cv.ViewOffset = start
-	cv.ViewSize = uint64(stop - start)
-}
-func (cv *ChunkView) Clone() IntervalValue {
-	return &ChunkView{
-		FileId:        cv.FileId,
-		OffsetInChunk: cv.OffsetInChunk,
-		ViewSize:      cv.ViewSize,
-		ViewOffset:    cv.ViewOffset,
-		ChunkSize:     cv.ChunkSize,
-		CipherKey:     cv.CipherKey,
-		IsGzipped:     cv.IsGzipped,
-		ModifiedTsNs:  cv.ModifiedTsNs,
-	}
+	FileId      string
+	Offset      int64
+	Size        uint64
+	LogicOffset int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
+	ChunkSize   uint64
+	CipherKey   []byte
+	IsGzipped   bool
 }
 
 func (cv *ChunkView) IsFullChunk() bool {
-	return cv.ViewSize == cv.ChunkSize
+	return cv.Size == cv.ChunkSize
 }
 
-func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
+func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {
 
 	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, offset, offset+size)
 
@@ -188,7 +152,7 @@ func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*
 
 }
 
-func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
+func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int64) (views []*ChunkView) {
 
 	stop := offset + size
 	if size == math.MaxInt64 {
@@ -198,112 +162,164 @@ func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset i
 		stop = math.MaxInt64
 	}
 
-	chunkViews = NewIntervalList[*ChunkView]()
-	for x := visibles.Front(); x != nil; x = x.Next {
-		chunk := x.Value
+	for _, chunk := range visibles {
 
 		chunkStart, chunkStop := max(offset, chunk.start), min(stop, chunk.stop)
 
 		if chunkStart < chunkStop {
-			chunkView := &ChunkView{
-				FileId:        chunk.fileId,
-				OffsetInChunk: chunkStart - chunk.start + chunk.offsetInChunk,
-				ViewSize:      uint64(chunkStop - chunkStart),
-				ViewOffset:    chunkStart,
-				ChunkSize:     chunk.chunkSize,
-				CipherKey:     chunk.cipherKey,
-				IsGzipped:     chunk.isGzipped,
-				ModifiedTsNs:  chunk.modifiedTsNs,
-			}
-			chunkViews.AppendInterval(&Interval[*ChunkView]{
-				StartOffset: chunkStart,
-				StopOffset:  chunkStop,
-				TsNs:        chunk.modifiedTsNs,
-				Value:       chunkView,
-				Prev:        nil,
-				Next:        nil,
+			views = append(views, &ChunkView{
+				FileId:      chunk.fileId,
+				Offset:      chunkStart - chunk.start + chunk.chunkOffset,
+				Size:        uint64(chunkStop - chunkStart),
+				LogicOffset: chunkStart,
+				ChunkSize:   chunk.chunkSize,
+				CipherKey:   chunk.cipherKey,
+				IsGzipped:   chunk.isGzipped,
 			})
 		}
 	}
 
-	return chunkViews
+	return views
 
 }
 
-func MergeIntoVisibles(visibles *IntervalList[*VisibleInterval], start int64, stop int64, chunk *filer_pb.FileChunk) {
+func logPrintf(name string, visibles []VisibleInterval) {
 
-	newV := &VisibleInterval{
-		start:         start,
-		stop:          stop,
-		fileId:        chunk.GetFileIdString(),
-		modifiedTsNs:  chunk.ModifiedTsNs,
-		offsetInChunk: start - chunk.Offset, // the starting position in the chunk
-		chunkSize:     chunk.Size,           // size of the chunk
-		cipherKey:     chunk.CipherKey,
-		isGzipped:     chunk.IsCompressed,
-	}
-
-	visibles.InsertInterval(start, stop, chunk.ModifiedTsNs, newV)
+	/*
+		glog.V(0).Infof("%s len %d", name, len(visibles))
+		for _, v := range visibles {
+			glog.V(0).Infof("%s:  [%d,%d) %s %d", name, v.start, v.stop, v.fileId, v.chunkOffset)
+		}
+	*/
 }
 
-func MergeIntoChunkViews(chunkViews *IntervalList[*ChunkView], start int64, stop int64, chunk *filer_pb.FileChunk) {
+func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (newVisibles []VisibleInterval) {
 
-	chunkView := &ChunkView{
-		FileId:        chunk.GetFileIdString(),
-		OffsetInChunk: start - chunk.Offset,
-		ViewSize:      uint64(stop - start),
-		ViewOffset:    start,
-		ChunkSize:     chunk.Size,
-		CipherKey:     chunk.CipherKey,
-		IsGzipped:     chunk.IsCompressed,
-		ModifiedTsNs:  chunk.ModifiedTsNs,
+	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.ModifiedTsNs, 0, chunk.Size, chunk.CipherKey, chunk.IsCompressed)
+
+	length := len(visibles)
+	if length == 0 {
+		return append(visibles, newV)
+	}
+	last := visibles[length-1]
+	if last.stop <= chunk.Offset {
+		return append(visibles, newV)
 	}
 
-	chunkViews.InsertInterval(start, stop, chunk.ModifiedTsNs, chunkView)
+	logPrintf("  before", visibles)
+	// glog.V(0).Infof("newVisibles %d adding chunk [%d,%d) %s size:%d", len(newVisibles), chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Size)
+	chunkStop := chunk.Offset + int64(chunk.Size)
+	for _, v := range visibles {
+		if v.start < chunk.Offset && chunk.Offset < v.stop {
+			t := newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTsNs, v.chunkOffset, v.chunkSize, v.cipherKey, v.isGzipped)
+			newVisibles = append(newVisibles, t)
+			// glog.V(0).Infof("visible %d [%d,%d) =1> [%d,%d)", i, v.start, v.stop, t.start, t.stop)
+		}
+		if v.start < chunkStop && chunkStop < v.stop {
+			t := newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTsNs, v.chunkOffset+(chunkStop-v.start), v.chunkSize, v.cipherKey, v.isGzipped)
+			newVisibles = append(newVisibles, t)
+			// glog.V(0).Infof("visible %d [%d,%d) =2> [%d,%d)", i, v.start, v.stop, t.start, t.stop)
+		}
+		if chunkStop <= v.start || v.stop <= chunk.Offset {
+			newVisibles = append(newVisibles, v)
+			// glog.V(0).Infof("visible %d [%d,%d) =3> [%d,%d)", i, v.start, v.stop, v.start, v.stop)
+		}
+	}
+	newVisibles = append(newVisibles, newV)
+
+	logPrintf("  append", newVisibles)
+
+	for i := len(newVisibles) - 1; i >= 0; i-- {
+		if i > 0 && newV.start < newVisibles[i-1].start {
+			newVisibles[i] = newVisibles[i-1]
+		} else {
+			newVisibles[i] = newV
+			break
+		}
+	}
+	logPrintf("  sorted", newVisibles)
+
+	return newVisibles
 }
 
 // NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory
 // If the file chunk content is a chunk manifest
-func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval], err error) {
+func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) {
 
 	chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)
 	if err != nil {
 		return
 	}
 
-	visibles2 := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles2 := readResolvedChunks(chunks)
 
-	return visibles2, err
+	if true {
+		return visibles2, err
+	}
+	slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
+		if a.ModifiedTsNs == b.ModifiedTsNs {
+			filer_pb.EnsureFid(a)
+			filer_pb.EnsureFid(b)
+			if a.Fid == nil || b.Fid == nil {
+				return true
+			}
+			return a.Fid.FileKey < b.Fid.FileKey
+		}
+		return a.ModifiedTsNs < b.ModifiedTsNs
+	})
+	for _, chunk := range chunks {
+
+		// glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
+		visibles = MergeIntoVisibles(visibles, chunk)
+
+		logPrintf("add", visibles)
+
+	}
+
+	if len(visibles) != len(visibles2) {
+		fmt.Printf("different visibles size %d : %d\n", len(visibles), len(visibles2))
+	} else {
+		for i := 0; i < len(visibles); i++ {
+			checkDifference(visibles[i], visibles2[i])
+		}
+	}
+
+	return
+}
+
+func checkDifference(x, y VisibleInterval) {
+	if x.start != y.start ||
+		x.stop != y.stop ||
+		x.fileId != y.fileId ||
+		x.modifiedTsNs != y.modifiedTsNs {
+		fmt.Printf("different visible %+v : %+v\n", x, y)
+	}
 }
 
 // find non-overlapping visible intervals
 // visible interval map to one file chunk
 
 type VisibleInterval struct {
-	start         int64
-	stop          int64
-	modifiedTsNs  int64
-	fileId        string
-	offsetInChunk int64
-	chunkSize     uint64
-	cipherKey     []byte
-	isGzipped     bool
+	start        int64
+	stop         int64
+	modifiedTsNs int64
+	fileId       string
+	chunkOffset  int64
+	chunkSize    uint64
+	cipherKey    []byte
+	isGzipped    bool
 }
 
-func (v *VisibleInterval) SetStartStop(start, stop int64) {
-	v.offsetInChunk += start - v.start
-	v.start, v.stop = start, stop
-}
-func (v *VisibleInterval) Clone() IntervalValue {
-	return &VisibleInterval{
-		start:         v.start,
-		stop:          v.stop,
-		modifiedTsNs:  v.modifiedTsNs,
-		fileId:        v.fileId,
-		offsetInChunk: v.offsetInChunk,
-		chunkSize:     v.chunkSize,
-		cipherKey:     v.cipherKey,
-		isGzipped:     v.isGzipped,
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, chunkOffset int64, chunkSize uint64, cipherKey []byte, isGzipped bool) VisibleInterval {
+	return VisibleInterval{
+		start:        start,
+		stop:         stop,
+		fileId:       fileId,
+		modifiedTsNs: modifiedTime,
+		chunkOffset:  chunkOffset, // the starting position in the chunk
+		chunkSize:    chunkSize,
+		cipherKey:    cipherKey,
+		isGzipped:    isGzipped,
 	}
 }
 

weed/filer/filechunks_read.go

@@ -1,22 +1,14 @@
 package filer
 
 import (
-	"container/list"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"golang.org/x/exp/slices"
 )
 
-func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval]) {
+func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
 
 	var points []*Point
 	for _, chunk := range chunks {
-		if chunk.IsChunkManifest {
-			println("This should not happen! A manifest chunk found:", chunk.GetFileIdString())
-		}
-		start, stop := max(chunk.Offset, startOffset), min(chunk.Offset+int64(chunk.Size), stopOffset)
-		if start >= stop {
-			continue
-		}
 		points = append(points, &Point{
 			x:       chunk.Offset,
 			ts:      chunk.ModifiedTsNs,
@@ -41,45 +33,40 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
 	})
 
 	var prevX int64
-	queue := list.New() // points with higher ts are at the tail
-	visibles = NewIntervalList[*VisibleInterval]()
-	var prevPoint *Point
+	var queue []*Point
 	for _, point := range points {
-		if queue.Len() > 0 {
-			prevPoint = queue.Back().Value.(*Point)
-		} else {
-			prevPoint = nil
-		}
 		if point.isStart {
-			if prevPoint != nil {
-				if point.x != prevX && prevPoint.ts < point.ts {
-					addToVisibles(visibles, prevX, prevPoint, point)
+			if len(queue) > 0 {
+				lastIndex := len(queue) - 1
+				lastPoint := queue[lastIndex]
+				if point.x != prevX && lastPoint.ts < point.ts {
+					visibles = addToVisibles(visibles, prevX, lastPoint, point)
 					prevX = point.x
 				}
 			}
 			// insert into queue
-			if prevPoint == nil || prevPoint.ts < point.ts {
-				queue.PushBack(point)
-				prevX = point.x
-			} else {
-				for e := queue.Front(); e != nil; e = e.Next() {
-					if e.Value.(*Point).ts > point.ts {
-						queue.InsertBefore(point, e)
-						break
+			for i := len(queue); i >= 0; i-- {
+				if i == 0 || queue[i-1].ts <= point.ts {
+					if i == len(queue) {
+						prevX = point.x
 					}
+					queue = addToQueue(queue, i, point)
+					break
 				}
 			}
 		} else {
-			isLast := true
-			for e := queue.Back(); e != nil; e = e.Prev() {
-				if e.Value.(*Point).ts == point.ts {
-					queue.Remove(e)
+			lastIndex := len(queue) - 1
+			index := lastIndex
+			var startPoint *Point
+			for ; index >= 0; index-- {
+				startPoint = queue[index]
+				if startPoint.ts == point.ts {
+					queue = removeFromQueue(queue, index)
 					break
 				}
-				isLast = false
 			}
-			if isLast && prevPoint != nil {
-				addToVisibles(visibles, prevX, prevPoint, point)
+			if index == lastIndex && startPoint != nil {
+				visibles = addToVisibles(visibles, prevX, startPoint, point)
 				prevX = point.x
 			}
 		}
@@ -88,30 +75,37 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
 	return
 }
 
-func addToVisibles(visibles *IntervalList[*VisibleInterval], prevX int64, startPoint *Point, point *Point) {
-	if prevX < point.x {
-		chunk := startPoint.chunk
-		visible := &VisibleInterval{
-			start:         prevX,
-			stop:          point.x,
-			fileId:        chunk.GetFileIdString(),
-			modifiedTsNs:  chunk.ModifiedTsNs,
-			offsetInChunk: prevX - chunk.Offset,
-			chunkSize:     chunk.Size,
-			cipherKey:     chunk.CipherKey,
-			isGzipped:     chunk.IsCompressed,
-		}
-		appendVisibleInterfal(visibles, visible)
+func removeFromQueue(queue []*Point, index int) []*Point {
+	for i := index; i < len(queue)-1; i++ {
+		queue[i] = queue[i+1]
 	}
+	queue = queue[:len(queue)-1]
+	return queue
 }
 
-func appendVisibleInterfal(visibles *IntervalList[*VisibleInterval], visible *VisibleInterval) {
-	visibles.AppendInterval(&Interval[*VisibleInterval]{
-		StartOffset: visible.start,
-		StopOffset:  visible.stop,
-		TsNs:        visible.modifiedTsNs,
-		Value:       visible,
-	})
+func addToQueue(queue []*Point, index int, point *Point) []*Point {
+	queue = append(queue, point)
+	for i := len(queue) - 1; i > index; i-- {
+		queue[i], queue[i-1] = queue[i-1], queue[i]
+	}
+	return queue
+}
+
+func addToVisibles(visibles []VisibleInterval, prevX int64, startPoint *Point, point *Point) []VisibleInterval {
+	if prevX < point.x {
+		chunk := startPoint.chunk
+		visibles = append(visibles, VisibleInterval{
+			start:        prevX,
+			stop:         point.x,
+			fileId:       chunk.GetFileIdString(),
+			modifiedTsNs: chunk.ModifiedTsNs,
+			chunkOffset:  prevX - chunk.Offset,
+			chunkSize:    chunk.Size,
+			cipherKey:    chunk.CipherKey,
+			isGzipped:    chunk.IsCompressed,
+		})
+	}
+	return visibles
 }
 
 type Point struct {

weed/filer/filechunks_read_test.go

@@ -3,7 +3,6 @@ package filer
 import (
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"math"
 	"math/rand"
 	"testing"
 )
@@ -43,38 +42,9 @@ func TestReadResolvedChunks(t *testing.T) {
 		},
 	}
 
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles := readResolvedChunks(chunks)
 
-	fmt.Printf("resolved to %d visible intervales\n", visibles.Len())
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
-		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
-	}
-
-}
-
-func TestReadResolvedChunks2(t *testing.T) {
-
-	chunks := []*filer_pb.FileChunk{
-		{
-			FileId:       "c",
-			Offset:       200,
-			Size:         50,
-			ModifiedTsNs: 3,
-		},
-		{
-			FileId:       "e",
-			Offset:       200,
-			Size:         25,
-			ModifiedTsNs: 5,
-		},
-	}
-
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
-
-	fmt.Printf("resolved to %d visible intervales\n", visibles.Len())
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
+	for _, visible := range visibles {
 		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
 	}
 
@@ -102,10 +72,9 @@ func TestRandomizedReadResolvedChunks(t *testing.T) {
 		chunks = append(chunks, randomWrite(array, start, size, ts))
 	}
 
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles := readResolvedChunks(chunks)
 
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
+	for _, visible := range visibles {
 		for i := visible.start; i < visible.stop; i++ {
 			if array[i] != visible.modifiedTsNs {
 				t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTsNs)
@@ -143,9 +112,9 @@ func TestSequentialReadResolvedChunks(t *testing.T) {
 		})
 	}
 
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles := readResolvedChunks(chunks)
 
-	fmt.Printf("visibles %d", visibles.Len())
+	fmt.Printf("visibles %d", len(visibles))
 
 }
 
@@ -232,48 +201,9 @@ func TestActualReadResolvedChunks(t *testing.T) {
 		},
 	}
 
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+	visibles := readResolvedChunks(chunks)
 
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
-		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
-	}
-
-}
-
-func TestActualReadResolvedChunks2(t *testing.T) {
-
-	chunks := []*filer_pb.FileChunk{
-		{
-			FileId:       "1,e7b96fef48",
-			Offset:       0,
-			Size:         184320,
-			ModifiedTsNs: 1,
-		},
-		{
-			FileId:       "2,22562640b9",
-			Offset:       184320,
-			Size:         4096,
-			ModifiedTsNs: 2,
-		},
-		{
-			FileId:       "2,33562640b9",
-			Offset:       184320,
-			Size:         4096,
-			ModifiedTsNs: 4,
-		},
-		{
-			FileId:       "4,df033e0fe4",
-			Offset:       188416,
-			Size:         2097152,
-			ModifiedTsNs: 3,
-		},
-	}
-
-	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
-
-	for x := visibles.Front(); x != nil; x = x.Next {
-		visible := x.Value
+	for _, visible := range visibles {
 		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
 	}
 

weed/filer/filechunks_test.go

@@ -92,8 +92,7 @@ func TestRandomFileChunksCompact(t *testing.T) {
 
 	visibles, _ := NonOverlappingVisibleIntervals(nil, chunks, 0, math.MaxInt64)
 
-	for visible := visibles.Front(); visible != nil; visible = visible.Next {
-		v := visible.Value
+	for _, v := range visibles {
 		for x := v.start; x < v.stop; x++ {
 			assert.Equal(t, strconv.Itoa(int(data[x])), v.fileId)
 		}
@@ -138,7 +137,7 @@ func TestIntervalMerging(t *testing.T) {
 			},
 			Expected: []*VisibleInterval{
 				{start: 0, stop: 70, fileId: "b"},
-				{start: 70, stop: 100, fileId: "a", offsetInChunk: 70},
+				{start: 70, stop: 100, fileId: "a", chunkOffset: 70},
 			},
 		},
 		// case 3: updates overwrite full chunks
@@ -175,15 +174,15 @@ func TestIntervalMerging(t *testing.T) {
 			},
 			Expected: []*VisibleInterval{
 				{start: 0, stop: 200, fileId: "d"},
-				{start: 200, stop: 220, fileId: "c", offsetInChunk: 130},
+				{start: 200, stop: 220, fileId: "c", chunkOffset: 130},
 			},
 		},
 		// case 6: same updates
 		{
 			Chunks: []*filer_pb.FileChunk{
 				{Offset: 0, Size: 100, FileId: "abc", Fid: &filer_pb.FileId{FileKey: 1}, ModifiedTsNs: 123},
-				{Offset: 0, Size: 100, FileId: "axf", Fid: &filer_pb.FileId{FileKey: 2}, ModifiedTsNs: 124},
-				{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, ModifiedTsNs: 125},
+				{Offset: 0, Size: 100, FileId: "axf", Fid: &filer_pb.FileId{FileKey: 2}, ModifiedTsNs: 123},
+				{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, ModifiedTsNs: 123},
 			},
 			Expected: []*VisibleInterval{
 				{start: 0, stop: 100, fileId: "xyz"},
@@ -229,17 +228,11 @@ func TestIntervalMerging(t *testing.T) {
 	for i, testcase := range testcases {
 		log.Printf("++++++++++ merged test case %d ++++++++++++++++++++", i)
 		intervals, _ := NonOverlappingVisibleIntervals(nil, testcase.Chunks, 0, math.MaxInt64)
-		x := -1
-		for visible := intervals.Front(); visible != nil; visible = visible.Next {
-			x++
-			interval := visible.Value
-			log.Printf("test case %d, interval start=%d, stop=%d, fileId=%s",
-				i, interval.start, interval.stop, interval.fileId)
+		for x, interval := range intervals {
+			log.Printf("test case %d, interval %d, start=%d, stop=%d, fileId=%s",
+				i, x, interval.start, interval.stop, interval.fileId)
 		}
-		x = -1
-		for visible := intervals.Front(); visible != nil; visible = visible.Next {
-			x++
-			interval := visible.Value
+		for x, interval := range intervals {
 			if interval.start != testcase.Expected[x].start {
 				t.Fatalf("failed on test case %d, interval %d, start %d, expect %d",
 					i, x, interval.start, testcase.Expected[x].start)
@@ -252,13 +245,13 @@ func TestIntervalMerging(t *testing.T) {
 				t.Fatalf("failed on test case %d, interval %d, chunkId %s, expect %s",
 					i, x, interval.fileId, testcase.Expected[x].fileId)
 			}
-			if interval.offsetInChunk != testcase.Expected[x].offsetInChunk {
-				t.Fatalf("failed on test case %d, interval %d, offsetInChunk %d, expect %d",
-					i, x, interval.offsetInChunk, testcase.Expected[x].offsetInChunk)
+			if interval.chunkOffset != testcase.Expected[x].chunkOffset {
+				t.Fatalf("failed on test case %d, interval %d, chunkOffset %d, expect %d",
+					i, x, interval.chunkOffset, testcase.Expected[x].chunkOffset)
 			}
 		}
-		if intervals.Len() != len(testcase.Expected) {
-			t.Fatalf("failed to compact test case %d, len %d expected %d", i, intervals.Len(), len(testcase.Expected))
+		if len(intervals) != len(testcase.Expected) {
+			t.Fatalf("failed to compact test case %d, len %d expected %d", i, len(intervals), len(testcase.Expected))
 		}
 
 	}
@@ -283,9 +276,9 @@ func TestChunksReading(t *testing.T) {
 			Offset: 0,
 			Size:   250,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 0, ViewSize: 100, FileId: "abc", ViewOffset: 0},
-				{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 100},
-				{OffsetInChunk: 0, ViewSize: 50, FileId: "fsad", ViewOffset: 200},
+				{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
+				{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
+				{Offset: 0, Size: 50, FileId: "fsad", LogicOffset: 200},
 			},
 		},
 		// case 1: updates overwrite full chunks
@@ -297,7 +290,7 @@ func TestChunksReading(t *testing.T) {
 			Offset: 50,
 			Size:   100,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 50, ViewSize: 100, FileId: "asdf", ViewOffset: 50},
+				{Offset: 50, Size: 100, FileId: "asdf", LogicOffset: 50},
 			},
 		},
 		// case 2: updates overwrite part of previous chunks
@@ -309,8 +302,8 @@ func TestChunksReading(t *testing.T) {
 			Offset: 30,
 			Size:   40,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 20, ViewSize: 30, FileId: "b", ViewOffset: 30},
-				{OffsetInChunk: 57, ViewSize: 10, FileId: "a", ViewOffset: 60},
+				{Offset: 20, Size: 30, FileId: "b", LogicOffset: 30},
+				{Offset: 57, Size: 10, FileId: "a", LogicOffset: 60},
 			},
 		},
 		// case 3: updates overwrite full chunks
@@ -323,8 +316,8 @@ func TestChunksReading(t *testing.T) {
 			Offset: 0,
 			Size:   200,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 0, ViewSize: 50, FileId: "asdf", ViewOffset: 0},
-				{OffsetInChunk: 0, ViewSize: 150, FileId: "xxxx", ViewOffset: 50},
+				{Offset: 0, Size: 50, FileId: "asdf", LogicOffset: 0},
+				{Offset: 0, Size: 150, FileId: "xxxx", LogicOffset: 50},
 			},
 		},
 		// case 4: updates far away from prev chunks
@@ -337,8 +330,8 @@ func TestChunksReading(t *testing.T) {
 			Offset: 0,
 			Size:   400,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 0, ViewSize: 200, FileId: "asdf", ViewOffset: 0},
-				{OffsetInChunk: 0, ViewSize: 150, FileId: "xxxx", ViewOffset: 250},
+				{Offset: 0, Size: 200, FileId: "asdf", LogicOffset: 0},
+				{Offset: 0, Size: 150, FileId: "xxxx", LogicOffset: 250},
 			},
 		},
 		// case 5: updates overwrite full chunks
@@ -352,21 +345,21 @@ func TestChunksReading(t *testing.T) {
 			Offset: 0,
 			Size:   220,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 0, ViewSize: 200, FileId: "c", ViewOffset: 0},
-				{OffsetInChunk: 130, ViewSize: 20, FileId: "b", ViewOffset: 200},
+				{Offset: 0, Size: 200, FileId: "c", LogicOffset: 0},
+				{Offset: 130, Size: 20, FileId: "b", LogicOffset: 200},
 			},
 		},
 		// case 6: same updates
 		{
 			Chunks: []*filer_pb.FileChunk{
 				{Offset: 0, Size: 100, FileId: "abc", Fid: &filer_pb.FileId{FileKey: 1}, ModifiedTsNs: 123},
-				{Offset: 0, Size: 100, FileId: "def", Fid: &filer_pb.FileId{FileKey: 2}, ModifiedTsNs: 124},
-				{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, ModifiedTsNs: 125},
+				{Offset: 0, Size: 100, FileId: "def", Fid: &filer_pb.FileId{FileKey: 2}, ModifiedTsNs: 123},
+				{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, ModifiedTsNs: 123},
 			},
 			Offset: 0,
 			Size:   100,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 0, ViewSize: 100, FileId: "xyz", ViewOffset: 0},
+				{Offset: 0, Size: 100, FileId: "xyz", LogicOffset: 0},
 			},
 		},
 		// case 7: edge cases
@@ -379,8 +372,8 @@ func TestChunksReading(t *testing.T) {
 			Offset: 0,
 			Size:   200,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 0, ViewSize: 100, FileId: "abc", ViewOffset: 0},
-				{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 100},
+				{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
+				{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
 			},
 		},
 		// case 8: edge cases
@@ -393,9 +386,9 @@ func TestChunksReading(t *testing.T) {
 			Offset: 0,
 			Size:   300,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 0, ViewSize: 90, FileId: "abc", ViewOffset: 0},
-				{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 90},
-				{OffsetInChunk: 0, ViewSize: 110, FileId: "fsad", ViewOffset: 190},
+				{Offset: 0, Size: 90, FileId: "abc", LogicOffset: 0},
+				{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 90},
+				{Offset: 0, Size: 110, FileId: "fsad", LogicOffset: 190},
 			},
 		},
 		// case 9: edge cases
@@ -411,12 +404,12 @@ func TestChunksReading(t *testing.T) {
 			Offset: 0,
 			Size:   153578836,
 			Expected: []*ChunkView{
-				{OffsetInChunk: 0, ViewSize: 43175936, FileId: "2,111fc2cbfac1", ViewOffset: 0},
-				{OffsetInChunk: 0, ViewSize: 52981760 - 43175936, FileId: "2,112a36ea7f85", ViewOffset: 43175936},
-				{OffsetInChunk: 0, ViewSize: 72564736 - 52981760, FileId: "4,112d5f31c5e7", ViewOffset: 52981760},
-				{OffsetInChunk: 0, ViewSize: 133255168 - 72564736, FileId: "1,113245f0cdb6", ViewOffset: 72564736},
-				{OffsetInChunk: 0, ViewSize: 137269248 - 133255168, FileId: "3,1141a70733b5", ViewOffset: 133255168},
-				{OffsetInChunk: 0, ViewSize: 153578836 - 137269248, FileId: "1,114201d5bbdb", ViewOffset: 137269248},
+				{Offset: 0, Size: 43175936, FileId: "2,111fc2cbfac1", LogicOffset: 0},
+				{Offset: 0, Size: 52981760 - 43175936, FileId: "2,112a36ea7f85", LogicOffset: 43175936},
+				{Offset: 0, Size: 72564736 - 52981760, FileId: "4,112d5f31c5e7", LogicOffset: 52981760},
+				{Offset: 0, Size: 133255168 - 72564736, FileId: "1,113245f0cdb6", LogicOffset: 72564736},
+				{Offset: 0, Size: 137269248 - 133255168, FileId: "3,1141a70733b5", LogicOffset: 133255168},
+				{Offset: 0, Size: 153578836 - 137269248, FileId: "1,114201d5bbdb", LogicOffset: 137269248},
 			},
 		},
 	}
@@ -427,31 +420,28 @@ func TestChunksReading(t *testing.T) {
 		}
 		log.Printf("++++++++++ read test case %d ++++++++++++++++++++", i)
 		chunks := ViewFromChunks(nil, testcase.Chunks, testcase.Offset, testcase.Size)
-		x := -1
-		for c := chunks.Front(); c != nil; c = c.Next {
-			x++
-			chunk := c.Value
+		for x, chunk := range chunks {
 			log.Printf("read case %d, chunk %d, offset=%d, size=%d, fileId=%s",
-				i, x, chunk.OffsetInChunk, chunk.ViewSize, chunk.FileId)
-			if chunk.OffsetInChunk != testcase.Expected[x].OffsetInChunk {
+				i, x, chunk.Offset, chunk.Size, chunk.FileId)
+			if chunk.Offset != testcase.Expected[x].Offset {
 				t.Fatalf("failed on read case %d, chunk %s, Offset %d, expect %d",
-					i, chunk.FileId, chunk.OffsetInChunk, testcase.Expected[x].OffsetInChunk)
+					i, chunk.FileId, chunk.Offset, testcase.Expected[x].Offset)
 			}
-			if chunk.ViewSize != testcase.Expected[x].ViewSize {
-				t.Fatalf("failed on read case %d, chunk %s, ViewSize %d, expect %d",
-					i, chunk.FileId, chunk.ViewSize, testcase.Expected[x].ViewSize)
+			if chunk.Size != testcase.Expected[x].Size {
+				t.Fatalf("failed on read case %d, chunk %s, Size %d, expect %d",
+					i, chunk.FileId, chunk.Size, testcase.Expected[x].Size)
 			}
 			if chunk.FileId != testcase.Expected[x].FileId {
 				t.Fatalf("failed on read case %d, chunk %d, FileId %s, expect %s",
 					i, x, chunk.FileId, testcase.Expected[x].FileId)
 			}
-			if chunk.ViewOffset != testcase.Expected[x].ViewOffset {
-				t.Fatalf("failed on read case %d, chunk %d, ViewOffset %d, expect %d",
-					i, x, chunk.ViewOffset, testcase.Expected[x].ViewOffset)
+			if chunk.LogicOffset != testcase.Expected[x].LogicOffset {
+				t.Fatalf("failed on read case %d, chunk %d, LogicOffset %d, expect %d",
+					i, x, chunk.LogicOffset, testcase.Expected[x].LogicOffset)
 			}
 		}
-		if chunks.Len() != len(testcase.Expected) {
-			t.Fatalf("failed to read test case %d, len %d expected %d", i, chunks.Len(), len(testcase.Expected))
+		if len(chunks) != len(testcase.Expected) {
+			t.Fatalf("failed to read test case %d, len %d expected %d", i, len(chunks), len(testcase.Expected))
 		}
 	}
 
@@ -477,79 +467,73 @@ func BenchmarkCompactFileChunks(b *testing.B) {
 	}
 }
 
-func addVisibleInterval(visibles *IntervalList[*VisibleInterval], x *VisibleInterval) {
-	visibles.AppendInterval(&Interval[*VisibleInterval]{
-		StartOffset: x.start,
-		StopOffset:  x.stop,
-		TsNs:        x.modifiedTsNs,
-		Value:       x,
-	})
-}
-
 func TestViewFromVisibleIntervals(t *testing.T) {
-	visibles := NewIntervalList[*VisibleInterval]()
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:  0,
-		stop:   25,
-		fileId: "fid1",
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:  4096,
-		stop:   8192,
-		fileId: "fid2",
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:  16384,
-		stop:   18551,
-		fileId: "fid3",
-	})
+	visibles := []VisibleInterval{
+		{
+			start:  0,
+			stop:   25,
+			fileId: "fid1",
+		},
+		{
+			start:  4096,
+			stop:   8192,
+			fileId: "fid2",
+		},
+		{
+			start:  16384,
+			stop:   18551,
+			fileId: "fid3",
+		},
+	}
 
 	views := ViewFromVisibleIntervals(visibles, 0, math.MaxInt32)
 
-	if views.Len() != visibles.Len() {
-		assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
+	if len(views) != len(visibles) {
+		assert.Equal(t, len(visibles), len(views), "ViewFromVisibleIntervals error")
 	}
 
 }
 
 func TestViewFromVisibleIntervals2(t *testing.T) {
-	visibles := NewIntervalList[*VisibleInterval]()
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:  344064,
-		stop:   348160,
-		fileId: "fid1",
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:  348160,
-		stop:   356352,
-		fileId: "fid2",
-	})
+	visibles := []VisibleInterval{
+		{
+			start:  344064,
+			stop:   348160,
+			fileId: "fid1",
+		},
+		{
+			start:  348160,
+			stop:   356352,
+			fileId: "fid2",
+		},
+	}
 
 	views := ViewFromVisibleIntervals(visibles, 0, math.MaxInt32)
 
-	if views.Len() != visibles.Len() {
-		assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
+	if len(views) != len(visibles) {
+		assert.Equal(t, len(visibles), len(views), "ViewFromVisibleIntervals error")
 	}
 
 }
 
 func TestViewFromVisibleIntervals3(t *testing.T) {
-	visibles := NewIntervalList[*VisibleInterval]()
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:  1000,
-		stop:   2000,
-		fileId: "fid1",
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:  3000,
-		stop:   4000,
-		fileId: "fid2",
-	})
+	visibles := []VisibleInterval{
+		{
+			start:  1000,
+			stop:   2000,
+			fileId: "fid1",
+		},
+		{
+			start:  3000,
+			stop:   4000,
+			fileId: "fid2",
+		},
+	}
 
 	views := ViewFromVisibleIntervals(visibles, 1700, 1500)
 
-	if views.Len() != visibles.Len() {
-		assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
+	if len(views) != len(visibles) {
+		assert.Equal(t, len(visibles), len(views), "ViewFromVisibleIntervals error")
 	}
 
 }

weed/filer/filer_conf.go

@@ -32,7 +32,7 @@ type FilerConf struct {
 
 func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) {
 	var buf bytes.Buffer
-	if err := pb.WithGrpcFilerClient(false, 0, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		if masterClient != nil {
 			return ReadEntry(masterClient, client, DirectoryEtcSeaweedFS, FilerConfName, &buf)
 		} else {

weed/filer/filer_notify.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"math"
-	"regexp"
 	"strings"
 	"time"
 
@@ -109,10 +108,6 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
 	}
 }
 
-var (
-	VolumeNotFoundPattern = regexp.MustCompile(`volume \d+? not found`)
-)
-
 func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, isDone bool, err error) {
 
 	startTime = startTime.UTC()
@@ -164,10 +159,6 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, each
 				if err == io.EOF {
 					continue
 				}
-				if VolumeNotFoundPattern.MatchString(err.Error()) {
-					glog.Warningf("skipping reading %s: %v", hourMinuteEntry.FullPath, err)
-					continue
-				}
 				return lastTsNs, isDone, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err)
 			}
 			chunkedFileReader.Close()

weed/filer/filer_notify_append.go

@@ -40,7 +40,7 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
 	}
 
 	// append to existing chunks
-	entry.Chunks = append(entry.GetChunks(), uploadResult.ToPbFileChunk(assignResult.Fid, offset, time.Now().UnixNano()))
+	entry.Chunks = append(entry.GetChunks(), uploadResult.ToPbFileChunk(assignResult.Fid, offset))
 
 	// update the entry
 	err = f.CreateEntry(context.Background(), entry, false, false, nil, false)

weed/filer/filerstore_wrapper.go

@@ -2,14 +2,13 @@ package filer
 
 import (
 	"context"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/viant/ptrie"
 	"io"
 	"math"
 	"strings"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/viant/ptrie"
-
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -187,12 +186,9 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
 	}
 	if len(existingEntry.HardLinkId) != 0 {
 		// remove hard link
-		op := ctx.Value("OP")
-		if op != "MV" {
-			glog.V(4).Infof("DeleteHardLink %s", existingEntry.FullPath)
-			if err = fsw.DeleteHardLink(ctx, existingEntry.HardLinkId); err != nil {
-				return err
-			}
+		glog.V(4).Infof("DeleteHardLink %s", existingEntry.FullPath)
+		if err = fsw.DeleteHardLink(ctx, existingEntry.HardLinkId); err != nil {
+			return err
 		}
 	}
 
@@ -210,12 +206,9 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
 
 	if len(existingEntry.HardLinkId) != 0 {
 		// remove hard link
-		op := ctx.Value("OP")
-		if op != "MV" {
-			glog.V(4).Infof("DeleteHardLink %s", existingEntry.FullPath)
-			if err = fsw.DeleteHardLink(ctx, existingEntry.HardLinkId); err != nil {
-				return err
-			}
+		glog.V(4).Infof("DeleteHardLink %s", existingEntry.FullPath)
+		if err = fsw.DeleteHardLink(ctx, existingEntry.HardLinkId); err != nil {
+			return err
 		}
 	}
 

weed/filer/interval_list.go

@@ -1,259 +0,0 @@
-package filer
-
-import (
-	"math"
-	"sync"
-)
-
-type IntervalValue interface {
-	SetStartStop(start, stop int64)
-	Clone() IntervalValue
-}
-
-type Interval[T IntervalValue] struct {
-	StartOffset int64
-	StopOffset  int64
-	TsNs        int64
-	Value       T
-	Prev        *Interval[T]
-	Next        *Interval[T]
-}
-
-func (interval *Interval[T]) Size() int64 {
-	return interval.StopOffset - interval.StartOffset
-}
-
-// IntervalList mark written intervals within one page chunk
-type IntervalList[T IntervalValue] struct {
-	head *Interval[T]
-	tail *Interval[T]
-	Lock sync.Mutex
-}
-
-func NewIntervalList[T IntervalValue]() *IntervalList[T] {
-	list := &IntervalList[T]{
-		head: &Interval[T]{
-			StartOffset: -1,
-			StopOffset:  -1,
-		},
-		tail: &Interval[T]{
-			StartOffset: math.MaxInt64,
-			StopOffset:  math.MaxInt64,
-		},
-	}
-	return list
-}
-
-func (list *IntervalList[T]) Front() (interval *Interval[T]) {
-	return list.head.Next
-}
-
-func (list *IntervalList[T]) AppendInterval(interval *Interval[T]) {
-	list.Lock.Lock()
-	defer list.Lock.Unlock()
-
-	if list.head.Next == nil {
-		list.head.Next = interval
-	}
-	interval.Prev = list.tail.Prev
-	if list.tail.Prev != nil {
-		list.tail.Prev.Next = interval
-	}
-	list.tail.Prev = interval
-}
-
-func (list *IntervalList[T]) Overlay(startOffset, stopOffset, tsNs int64, value T) {
-	if startOffset >= stopOffset {
-		return
-	}
-	interval := &Interval[T]{
-		StartOffset: startOffset,
-		StopOffset:  stopOffset,
-		TsNs:        tsNs,
-		Value:       value,
-	}
-
-	list.Lock.Lock()
-	defer list.Lock.Unlock()
-
-	list.overlayInterval(interval)
-}
-
-func (list *IntervalList[T]) InsertInterval(startOffset, stopOffset, tsNs int64, value T) {
-	interval := &Interval[T]{
-		StartOffset: startOffset,
-		StopOffset:  stopOffset,
-		TsNs:        tsNs,
-		Value:       value,
-	}
-
-	list.Lock.Lock()
-	defer list.Lock.Unlock()
-
-	value.SetStartStop(startOffset, stopOffset)
-	list.insertInterval(interval)
-}
-
-func (list *IntervalList[T]) insertInterval(interval *Interval[T]) {
-	prev := list.head
-	next := prev.Next
-
-	for interval.StartOffset < interval.StopOffset {
-		if next == nil {
-			// add to the end
-			list.insertBetween(prev, interval, list.tail)
-			break
-		}
-
-		// interval is ahead of the next
-		if interval.StopOffset <= next.StartOffset {
-			list.insertBetween(prev, interval, next)
-			break
-		}
-
-		// interval is after the next
-		if next.StopOffset <= interval.StartOffset {
-			prev = next
-			next = next.Next
-			continue
-		}
-
-		// intersecting next and interval
-		if interval.TsNs >= next.TsNs {
-			// interval is newer
-			if next.StartOffset < interval.StartOffset {
-				// left side of next is ahead of interval
-				t := &Interval[T]{
-					StartOffset: next.StartOffset,
-					StopOffset:  interval.StartOffset,
-					TsNs:        next.TsNs,
-					Value:       next.Value.Clone().(T),
-				}
-				t.Value.SetStartStop(t.StartOffset, t.StopOffset)
-				list.insertBetween(prev, t, interval)
-				next.StartOffset = interval.StartOffset
-				next.Value.SetStartStop(next.StartOffset, next.StopOffset)
-				prev = t
-			}
-			if interval.StopOffset < next.StopOffset {
-				// right side of next is after interval
-				next.StartOffset = interval.StopOffset
-				next.Value.SetStartStop(next.StartOffset, next.StopOffset)
-				list.insertBetween(prev, interval, next)
-				break
-			} else {
-				// next is covered
-				prev.Next = interval
-				next = next.Next
-			}
-		} else {
-			// next is newer
-			if interval.StartOffset < next.StartOffset {
-				// left side of interval is ahead of next
-				t := &Interval[T]{
-					StartOffset: interval.StartOffset,
-					StopOffset:  next.StartOffset,
-					TsNs:        interval.TsNs,
-					Value:       interval.Value.Clone().(T),
-				}
-				t.Value.SetStartStop(t.StartOffset, t.StopOffset)
-				list.insertBetween(prev, t, next)
-				interval.StartOffset = next.StartOffset
-				interval.Value.SetStartStop(interval.StartOffset, interval.StopOffset)
-			}
-			if next.StopOffset < interval.StopOffset {
-				// right side of interval is after next
-				interval.StartOffset = next.StopOffset
-				interval.Value.SetStartStop(interval.StartOffset, interval.StopOffset)
-			} else {
-				// interval is covered
-				break
-			}
-		}
-
-	}
-}
-
-func (list *IntervalList[T]) insertBetween(a, interval, b *Interval[T]) {
-	a.Next = interval
-	b.Prev = interval
-	if a != list.head {
-		interval.Prev = a
-	}
-	if b != list.tail {
-		interval.Next = b
-	}
-}
-
-func (list *IntervalList[T]) overlayInterval(interval *Interval[T]) {
-
-	//t := list.head
-	//for ; t.Next != nil; t = t.Next {
-	//	if t.TsNs > interval.TsNs {
-	//		println("writes is out of order", t.TsNs-interval.TsNs, "ns")
-	//	}
-	//}
-
-	p := list.head
-	for ; p.Next != nil && p.Next.StopOffset <= interval.StartOffset; p = p.Next {
-	}
-	q := list.tail
-	for ; q.Prev != nil && q.Prev.StartOffset >= interval.StopOffset; q = q.Prev {
-	}
-
-	// left side
-	// interval after p.Next start
-	if p.Next != nil && p.Next.StartOffset < interval.StartOffset {
-		t := &Interval[T]{
-			StartOffset: p.Next.StartOffset,
-			StopOffset:  interval.StartOffset,
-			TsNs:        p.Next.TsNs,
-			Value:       p.Next.Value,
-		}
-		p.Next = t
-		if p != list.head {
-			t.Prev = p
-		}
-		t.Next = interval
-		interval.Prev = t
-	} else {
-		p.Next = interval
-		if p != list.head {
-			interval.Prev = p
-		}
-	}
-
-	// right side
-	// interval ends before p.Prev
-	if q.Prev != nil && interval.StopOffset < q.Prev.StopOffset {
-		t := &Interval[T]{
-			StartOffset: interval.StopOffset,
-			StopOffset:  q.Prev.StopOffset,
-			TsNs:        q.Prev.TsNs,
-			Value:       q.Prev.Value,
-		}
-		q.Prev = t
-		if q != list.tail {
-			t.Next = q
-		}
-		interval.Next = t
-		t.Prev = interval
-	} else {
-		q.Prev = interval
-		if q != list.tail {
-			interval.Next = q
-		}
-	}
-
-}
-
-func (list *IntervalList[T]) Len() int {
-	list.Lock.Lock()
-	defer list.Lock.Unlock()
-
-	var count int
-	for t := list.head; t != nil; t = t.Next {
-		count++
-	}
-	return count - 1
-}

weed/filer/interval_list_test.go

@@ -1,327 +0,0 @@
-package filer
-
-import (
-	"fmt"
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-type IntervalInt int
-
-func (i IntervalInt) SetStartStop(start, stop int64) {
-}
-func (i IntervalInt) Clone() IntervalValue {
-	return i
-}
-
-func TestIntervalList_Overlay(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(0, 100, 1, 1)
-	list.Overlay(50, 150, 2, 2)
-	list.Overlay(200, 250, 3, 3)
-	list.Overlay(225, 250, 4, 4)
-	list.Overlay(175, 210, 5, 5)
-	list.Overlay(0, 25, 6, 6)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 6, list.Len())
-	println()
-	list.Overlay(50, 150, 7, 7)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 6, list.Len())
-}
-
-func TestIntervalList_Overlay2(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	list.Overlay(0, 50, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-}
-
-func TestIntervalList_Overlay3(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	assert.Equal(t, 1, list.Len())
-
-	list.Overlay(0, 60, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_Overlay4(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	list.Overlay(0, 100, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 1, list.Len())
-}
-
-func TestIntervalList_Overlay5(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	list.Overlay(0, 110, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 1, list.Len())
-}
-
-func TestIntervalList_Overlay6(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	list.Overlay(50, 110, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 1, list.Len())
-}
-
-func TestIntervalList_Overlay7(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	list.Overlay(50, 90, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_Overlay8(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	list.Overlay(60, 90, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 3, list.Len())
-}
-
-func TestIntervalList_Overlay9(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	list.Overlay(60, 100, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_Overlay10(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(50, 100, 1, 1)
-	list.Overlay(60, 110, 2, 2)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_Overlay11(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.Overlay(0, 100, 1, 1)
-	list.Overlay(100, 110, 2, 2)
-	list.Overlay(0, 90, 3, 3)
-	list.Overlay(0, 80, 4, 4)
-	list.Overlay(0, 90, 5, 5)
-	list.Overlay(90, 90, 6, 6)
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 3, list.Len())
-}
-
-func TestIntervalList_insertInterval1(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.InsertInterval(50, 150, 2, 2)
-	list.InsertInterval(200, 250, 3, 3)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_insertInterval2(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.InsertInterval(50, 150, 2, 2)
-	list.InsertInterval(0, 25, 3, 3)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_insertInterval3(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.InsertInterval(50, 150, 2, 2)
-	list.InsertInterval(200, 250, 4, 4)
-
-	list.InsertInterval(0, 75, 3, 3)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 3, list.Len())
-}
-
-func TestIntervalList_insertInterval4(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.InsertInterval(200, 250, 4, 4)
-
-	list.InsertInterval(0, 225, 3, 3)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_insertInterval5(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-	list.InsertInterval(200, 250, 4, 4)
-
-	list.InsertInterval(0, 225, 5, 5)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_insertInterval6(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-
-	list.InsertInterval(50, 150, 2, 2)
-	list.InsertInterval(200, 250, 4, 4)
-
-	list.InsertInterval(0, 275, 1, 1)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 5, list.Len())
-}
-
-func TestIntervalList_insertInterval7(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-
-	list.InsertInterval(50, 150, 2, 2)
-	list.InsertInterval(200, 250, 4, 4)
-
-	list.InsertInterval(75, 275, 1, 1)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 4, list.Len())
-}
-
-func TestIntervalList_insertInterval8(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-
-	list.InsertInterval(50, 150, 2, 2)
-	list.InsertInterval(200, 250, 4, 4)
-
-	list.InsertInterval(75, 275, 3, 3)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 4, list.Len())
-}
-
-func TestIntervalList_insertInterval9(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-
-	list.InsertInterval(50, 150, 2, 2)
-	list.InsertInterval(200, 250, 4, 4)
-
-	list.InsertInterval(50, 150, 3, 3)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 2, list.Len())
-}
-
-func TestIntervalList_insertInterval10(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-
-	list.InsertInterval(50, 100, 2, 2)
-
-	list.InsertInterval(200, 300, 4, 4)
-
-	list.InsertInterval(100, 200, 5, 5)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 3, list.Len())
-}
-
-func TestIntervalList_insertInterval11(t *testing.T) {
-	list := NewIntervalList[IntervalInt]()
-
-	list.InsertInterval(0, 64, 1, 1)
-
-	list.InsertInterval(72, 136, 3, 3)
-
-	list.InsertInterval(64, 128, 2, 2)
-
-	list.InsertInterval(68, 72, 4, 4)
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 4, list.Len())
-}
-
-type IntervalStruct struct {
-	x     int
-	start int64
-	stop  int64
-}
-
-func newIntervalStruct(i int) IntervalStruct {
-	return IntervalStruct{
-		x: i,
-	}
-}
-
-func (i IntervalStruct) SetStartStop(start, stop int64) {
-	i.start, i.stop = start, stop
-}
-func (i IntervalStruct) Clone() IntervalValue {
-	return &IntervalStruct{
-		x:     i.x,
-		start: i.start,
-		stop:  i.stop,
-	}
-}
-
-func TestIntervalList_insertIntervalStruct(t *testing.T) {
-	list := NewIntervalList[IntervalStruct]()
-
-	list.InsertInterval(0, 64, 1, newIntervalStruct(1))
-
-	list.InsertInterval(64, 72, 2, newIntervalStruct(2))
-
-	list.InsertInterval(72, 136, 3, newIntervalStruct(3))
-
-	list.InsertInterval(64, 68, 4, newIntervalStruct(4))
-
-	for p := list.Front(); p != nil; p = p.Next {
-		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
-	}
-	assert.Equal(t, 4, list.Len())
-}

weed/filer/meta_aggregator.go

@@ -192,7 +192,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
 	}
 
 	glog.V(0).Infof("subscribing remote %s meta change: %v, clientId:%d", peer, time.Unix(0, lastTsNs), ma.filer.UniqueFilerId)
-	err = pb.WithFilerClient(true, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithFilerClient(true, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 		atomic.AddInt32(&ma.filer.UniqueFilerEpoch, 1)
@@ -228,7 +228,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
 }
 
 func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
-	err = pb.WithFilerClient(false, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
 			return err
@@ -243,15 +243,10 @@ const (
 	MetaOffsetPrefix = "Meta"
 )
 
-func GetPeerMetaOffsetKey(peerSignature int32) []byte {
-	key := []byte(MetaOffsetPrefix + "xxxx")
-	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))
-	return key
-}
-
 func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignature int32) (lastTsNs int64, err error) {
 
-	key := GetPeerMetaOffsetKey(peerSignature)
+	key := []byte(MetaOffsetPrefix + "xxxx")
+	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))
 
 	value, err := f.Store.KvGet(context.Background(), key)
 
@@ -268,7 +263,8 @@ func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignat
 
 func (ma *MetaAggregator) updateOffset(f *Filer, peer pb.ServerAddress, peerSignature int32, lastTsNs int64) (err error) {
 
-	key := GetPeerMetaOffsetKey(peerSignature)
+	key := []byte(MetaOffsetPrefix + "xxxx")
+	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))
 
 	value := make([]byte, 8)
 	util.Uint64toBytes(value, uint64(lastTsNs))

weed/filer/mysql/mysql_sql_gen.go

@@ -21,32 +21,32 @@ func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
 	if gen.UpsertQueryTemplate != "" {
 		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
 	} else {
-		return fmt.Sprintf("INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES(?,?,?,?)", tableName)
+		return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
 	}
 }
 
 func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {
-	return fmt.Sprintf("UPDATE `%s` SET `meta` = ? WHERE `dirhash` = ? AND `name` = ? AND `directory` = ?", tableName)
+	return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlFind(tableName string) string {
-	return fmt.Sprintf("SELECT `meta` FROM `%s` WHERE `dirhash` = ? AND `name` = ? AND `directory` = ?", tableName)
+	return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE `dirhash` = ? AND `name` = ? AND `directory` = ?", tableName)
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE `dirhash` = ? AND `directory` = ?", tableName)
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
-	return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND `name` > ? AND `directory` = ? AND `name` LIKE ? ORDER BY `name` ASC LIMIT ?", tableName)
+	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
-	return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND `name` >= ? AND `directory` = ? AND `name` LIKE ? ORDER BY `name` ASC LIMIT ?", tableName)
+	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {

weed/filer/mysql/mysql_store.go

@@ -13,7 +13,7 @@ import (
 )
 
 const (
-	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?collation=utf8mb4_bin"
+	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
 )
 
 func init() {
@@ -53,7 +53,7 @@ func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user,
 	}
 	store.SqlGenerator = &SqlGenMysql{
 		CreateTableSqlTemplate: "",
-		DropTableSqlTemplate:   "DROP TABLE `%s`",
+		DropTableSqlTemplate:   "drop table `%s`",
 		UpsertQueryTemplate:    upsertQuery,
 	}
 

weed/filer/mysql2/mysql2_store.go

@@ -15,7 +15,7 @@ import (
 )
 
 const (
-	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?collation=utf8mb4_bin"
+	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
 )
 
 var _ filer.BucketAware = (*MysqlStore2)(nil)
@@ -58,7 +58,7 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse
 	}
 	store.SqlGenerator = &mysql.SqlGenMysql{
 		CreateTableSqlTemplate: createTable,
-		DropTableSqlTemplate:   "DROP TABLE `%s`",
+		DropTableSqlTemplate:   "drop table `%s`",
 		UpsertQueryTemplate:    upsertQuery,
 	}
 

weed/filer/reader_at.go

@@ -10,12 +10,14 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 )
 
 type ChunkReadAt struct {
 	masterClient  *wdclient.MasterClient
-	chunkViews    *IntervalList[*ChunkView]
+	chunkViews    []*ChunkView
+	readerLock    sync.Mutex
 	fileSize      int64
 	readerCache   *ReaderCache
 	readerPattern *ReaderPattern
@@ -87,12 +89,12 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp
 	}
 }
 
-func NewChunkReaderAtFromClient(readerCache *ReaderCache, chunkViews *IntervalList[*ChunkView], fileSize int64) *ChunkReadAt {
+func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt {
 
 	return &ChunkReadAt{
 		chunkViews:    chunkViews,
 		fileSize:      fileSize,
-		readerCache:   readerCache,
+		readerCache:   newReaderCache(32, chunkCache, lookupFn),
 		readerPattern: NewReaderPattern(),
 	}
 }
@@ -106,58 +108,44 @@ func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) {
 
 	c.readerPattern.MonitorReadAt(offset, len(p))
 
-	c.chunkViews.Lock.Lock()
-	defer c.chunkViews.Lock.Unlock()
-
-	// glog.V(4).Infof("ReadAt [%d,%d) of total file size %d bytes %d chunk views", offset, offset+int64(len(p)), c.fileSize, len(c.chunkViews))
-	n, _, err = c.doReadAt(p, offset)
-	return
-}
-
-func (c *ChunkReadAt) ReadAtWithTime(p []byte, offset int64) (n int, ts int64, err error) {
-
-	c.readerPattern.MonitorReadAt(offset, len(p))
-
-	c.chunkViews.Lock.Lock()
-	defer c.chunkViews.Lock.Unlock()
+	c.readerLock.Lock()
+	defer c.readerLock.Unlock()
 
 	// glog.V(4).Infof("ReadAt [%d,%d) of total file size %d bytes %d chunk views", offset, offset+int64(len(p)), c.fileSize, len(c.chunkViews))
 	return c.doReadAt(p, offset)
 }
 
-func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, ts int64, err error) {
+func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {
 
 	startOffset, remaining := offset, int64(len(p))
-	var nextChunks *Interval[*ChunkView]
-	for x := c.chunkViews.Front(); x != nil; x = x.Next {
-		chunk := x.Value
+	var nextChunks []*ChunkView
+	for i, chunk := range c.chunkViews {
 		if remaining <= 0 {
 			break
 		}
-		if x.Next != nil {
-			nextChunks = x.Next
+		if i+1 < len(c.chunkViews) {
+			nextChunks = c.chunkViews[i+1:]
 		}
-		if startOffset < chunk.ViewOffset {
-			gap := chunk.ViewOffset - startOffset
-			glog.V(4).Infof("zero [%d,%d)", startOffset, chunk.ViewOffset)
+		if startOffset < chunk.LogicOffset {
+			gap := chunk.LogicOffset - startOffset
+			glog.V(4).Infof("zero [%d,%d)", startOffset, chunk.LogicOffset)
 			n += zero(p, startOffset-offset, gap)
-			startOffset, remaining = chunk.ViewOffset, remaining-gap
+			startOffset, remaining = chunk.LogicOffset, remaining-gap
 			if remaining <= 0 {
 				break
 			}
 		}
-		// fmt.Printf(">>> doReadAt [%d,%d), chunk[%d,%d)\n", offset, offset+int64(len(p)), chunk.ViewOffset, chunk.ViewOffset+int64(chunk.ViewSize))
-		chunkStart, chunkStop := max(chunk.ViewOffset, startOffset), min(chunk.ViewOffset+int64(chunk.ViewSize), startOffset+remaining)
+		// fmt.Printf(">>> doReadAt [%d,%d), chunk[%d,%d)\n", offset, offset+int64(len(p)), chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
+		chunkStart, chunkStop := max(chunk.LogicOffset, startOffset), min(chunk.LogicOffset+int64(chunk.Size), startOffset+remaining)
 		if chunkStart >= chunkStop {
 			continue
 		}
-		// glog.V(4).Infof("read [%d,%d), %d/%d chunk %s [%d,%d)", chunkStart, chunkStop, i, len(c.chunkViews), chunk.FileId, chunk.ViewOffset-chunk.Offset, chunk.ViewOffset-chunk.Offset+int64(chunk.ViewSize))
-		bufferOffset := chunkStart - chunk.ViewOffset + chunk.OffsetInChunk
-		ts = chunk.ModifiedTsNs
+		// glog.V(4).Infof("read [%d,%d), %d/%d chunk %s [%d,%d)", chunkStart, chunkStop, i, len(c.chunkViews), chunk.FileId, chunk.LogicOffset-chunk.Offset, chunk.LogicOffset-chunk.Offset+int64(chunk.Size))
+		bufferOffset := chunkStart - chunk.LogicOffset + chunk.Offset
 		copied, err := c.readChunkSliceAt(p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], chunk, nextChunks, uint64(bufferOffset))
 		if err != nil {
 			glog.Errorf("fetching chunk %+v: %v\n", chunk, err)
-			return copied, ts, err
+			return copied, err
 		}
 
 		n += copied
@@ -189,7 +177,7 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, ts int64, err err
 
 }
 
-func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, nextChunkViews *Interval[*ChunkView], offset uint64) (n int, err error) {
+func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, nextChunkViews []*ChunkView, offset uint64) (n int, err error) {
 
 	if c.readerPattern.IsRandomMode() {
 		n, err := c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
@@ -199,14 +187,16 @@ func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, next
 		return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset))
 	}
 
-	n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), chunkView.ViewOffset == 0)
+	n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), chunkView.LogicOffset == 0)
 	if c.lastChunkFid != chunkView.FileId {
-		if chunkView.OffsetInChunk == 0 { // start of a new chunk
+		if chunkView.Offset == 0 { // start of a new chunk
 			if c.lastChunkFid != "" {
 				c.readerCache.UnCache(c.lastChunkFid)
-			}
-			if nextChunkViews != nil {
-				c.readerCache.MaybeCache(nextChunkViews) // just read the next chunk if at the very beginning
+				c.readerCache.MaybeCache(nextChunkViews)
+			} else {
+				if len(nextChunkViews) >= 1 {
+					c.readerCache.MaybeCache(nextChunkViews[:1]) // just read the next chunk if at the very beginning
+				}
 			}
 		}
 	}

weed/filer/reader_at_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"math"
 	"strconv"
+	"sync"
 	"testing"
 )
 
@@ -33,42 +34,44 @@ func (m *mockChunkCache) SetChunk(fileId string, data []byte) {
 
 func TestReaderAt(t *testing.T) {
 
-	visibles := NewIntervalList[*VisibleInterval]()
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     1,
-		stop:      2,
-		fileId:    "1",
-		chunkSize: 9,
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     3,
-		stop:      4,
-		fileId:    "3",
-		chunkSize: 1,
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     5,
-		stop:      6,
-		fileId:    "5",
-		chunkSize: 2,
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     7,
-		stop:      9,
-		fileId:    "7",
-		chunkSize: 2,
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     9,
-		stop:      10,
-		fileId:    "9",
-		chunkSize: 2,
-	})
+	visibles := []VisibleInterval{
+		{
+			start:     1,
+			stop:      2,
+			fileId:    "1",
+			chunkSize: 9,
+		},
+		{
+			start:     3,
+			stop:      4,
+			fileId:    "3",
+			chunkSize: 1,
+		},
+		{
+			start:     5,
+			stop:      6,
+			fileId:    "5",
+			chunkSize: 2,
+		},
+		{
+			start:     7,
+			stop:      9,
+			fileId:    "7",
+			chunkSize: 2,
+		},
+		{
+			start:     9,
+			stop:      10,
+			fileId:    "9",
+			chunkSize: 2,
+		},
+	}
 
 	readerAt := &ChunkReadAt{
 		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
+		readerLock:    sync.Mutex{},
 		fileSize:      10,
-		readerCache:   NewReaderCache(3, &mockChunkCache{}, nil),
+		readerCache:   newReaderCache(3, &mockChunkCache{}, nil),
 		readerPattern: NewReaderPattern(),
 	}
 
@@ -83,7 +86,7 @@ func testReadAt(t *testing.T, readerAt *ChunkReadAt, offset int64, size int, exp
 	if data == nil {
 		data = make([]byte, size)
 	}
-	n, _, err := readerAt.doReadAt(data, offset)
+	n, err := readerAt.doReadAt(data, offset)
 
 	if expectedN != n {
 		t.Errorf("unexpected read size: %d, expect: %d", n, expectedN)
@@ -98,24 +101,26 @@ func testReadAt(t *testing.T, readerAt *ChunkReadAt, offset int64, size int, exp
 
 func TestReaderAt0(t *testing.T) {
 
-	visibles := NewIntervalList[*VisibleInterval]()
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     2,
-		stop:      5,
-		fileId:    "1",
-		chunkSize: 9,
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     7,
-		stop:      9,
-		fileId:    "2",
-		chunkSize: 9,
-	})
+	visibles := []VisibleInterval{
+		{
+			start:     2,
+			stop:      5,
+			fileId:    "1",
+			chunkSize: 9,
+		},
+		{
+			start:     7,
+			stop:      9,
+			fileId:    "2",
+			chunkSize: 9,
+		},
+	}
 
 	readerAt := &ChunkReadAt{
 		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
+		readerLock:    sync.Mutex{},
 		fileSize:      10,
-		readerCache:   NewReaderCache(3, &mockChunkCache{}, nil),
+		readerCache:   newReaderCache(3, &mockChunkCache{}, nil),
 		readerPattern: NewReaderPattern(),
 	}
 
@@ -130,18 +135,20 @@ func TestReaderAt0(t *testing.T) {
 
 func TestReaderAt1(t *testing.T) {
 
-	visibles := NewIntervalList[*VisibleInterval]()
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     2,
-		stop:      5,
-		fileId:    "1",
-		chunkSize: 9,
-	})
+	visibles := []VisibleInterval{
+		{
+			start:     2,
+			stop:      5,
+			fileId:    "1",
+			chunkSize: 9,
+		},
+	}
 
 	readerAt := &ChunkReadAt{
 		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
+		readerLock:    sync.Mutex{},
 		fileSize:      20,
-		readerCache:   NewReaderCache(3, &mockChunkCache{}, nil),
+		readerCache:   newReaderCache(3, &mockChunkCache{}, nil),
 		readerPattern: NewReaderPattern(),
 	}
 
@@ -157,24 +164,26 @@ func TestReaderAt1(t *testing.T) {
 }
 
 func TestReaderAtGappedChunksDoNotLeak(t *testing.T) {
-	visibles := NewIntervalList[*VisibleInterval]()
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     2,
-		stop:      3,
-		fileId:    "1",
-		chunkSize: 5,
-	})
-	addVisibleInterval(visibles, &VisibleInterval{
-		start:     7,
-		stop:      9,
-		fileId:    "1",
-		chunkSize: 4,
-	})
+	visibles := []VisibleInterval{
+		{
+			start:     2,
+			stop:      3,
+			fileId:    "1",
+			chunkSize: 5,
+		},
+		{
+			start:     7,
+			stop:      9,
+			fileId:    "1",
+			chunkSize: 4,
+		},
+	}
 
 	readerAt := &ChunkReadAt{
 		chunkViews:    ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
+		readerLock:    sync.Mutex{},
 		fileSize:      9,
-		readerCache:   NewReaderCache(3, &mockChunkCache{}, nil),
+		readerCache:   newReaderCache(3, &mockChunkCache{}, nil),
 		readerPattern: NewReaderPattern(),
 	}
 
@@ -184,9 +193,10 @@ func TestReaderAtGappedChunksDoNotLeak(t *testing.T) {
 
 func TestReaderAtSparseFileDoesNotLeak(t *testing.T) {
 	readerAt := &ChunkReadAt{
-		chunkViews:    ViewFromVisibleIntervals(NewIntervalList[*VisibleInterval](), 0, math.MaxInt64),
+		chunkViews:    ViewFromVisibleIntervals([]VisibleInterval{}, 0, math.MaxInt64),
+		readerLock:    sync.Mutex{},
 		fileSize:      3,
-		readerCache:   NewReaderCache(3, &mockChunkCache{}, nil),
+		readerCache:   newReaderCache(3, &mockChunkCache{}, nil),
 		readerPattern: NewReaderPattern(),
 	}
 

weed/filer/reader_cache.go

@@ -34,7 +34,7 @@ type SingleChunkCacher struct {
 	completedTimeNew int64
 }
 
-func NewReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType) *ReaderCache {
+func newReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType) *ReaderCache {
 	return &ReaderCache{
 		limit:          limit,
 		chunkCache:     chunkCache,
@@ -43,7 +43,7 @@ func NewReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn
 	}
 }
 
-func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView]) {
+func (rc *ReaderCache) MaybeCache(chunkViews []*ChunkView) {
 	if rc.lookupFileIdFn == nil {
 		return
 	}
@@ -55,8 +55,7 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView]) {
 		return
 	}
 
-	for x := chunkViews; x != nil; x = x.Next {
-		chunkView := x.Value
+	for _, chunkView := range chunkViews {
 		if _, found := rc.downloaders[chunkView.FileId]; found {
 			continue
 		}
@@ -66,7 +65,7 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView]) {
 			return
 		}
 
-		// glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.ViewOffset)
+		// glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.LogicOffset)
 		// cache this chunk if not yet
 		cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), false)
 		go cacher.startCaching()
@@ -208,7 +207,7 @@ func (s *SingleChunkCacher) readChunkAt(buf []byte, offset int64) (int, error) {
 		return 0, s.err
 	}
 
-	if len(s.data) <= int(offset) {
+	if len(s.data) == 0 {
 		return 0, nil
 	}
 

weed/filer/remote_mapping.go

@@ -11,7 +11,7 @@ import (
 
 func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress) (mappings *remote_pb.RemoteStorageMapping, readErr error) {
 	var oldContent []byte
-	if readErr = pb.WithFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if readErr = pb.WithFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
 		return readErr
 	}); readErr != nil {

weed/filer/remote_storage.go

@@ -133,7 +133,7 @@ func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *remote_pb.Remo
 
 func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, storageName string) (conf *remote_pb.RemoteConf, readErr error) {
 	var oldContent []byte
-	if readErr = pb.WithFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if readErr = pb.WithFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX)
 		return readErr
 	}); readErr != nil {

weed/filer/stream.go

@@ -6,6 +6,7 @@ import (
 	"golang.org/x/exp/slices"
 	"io"
 	"math"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -77,8 +78,7 @@ func StreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, w
 
 	fileId2Url := make(map[string][]string)
 
-	for x := chunkViews.Front(); x != nil; x = x.Next {
-		chunkView := x.Value
+	for _, chunkView := range chunkViews {
 		var urlStrings []string
 		var err error
 		for _, backoff := range getLookupFileIdBackoffSchedule {
@@ -102,30 +102,29 @@ func StreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, w
 
 	downloadThrottler := util.NewWriteThrottler(downloadMaxBytesPs)
 	remaining := size
-	for x := chunkViews.Front(); x != nil; x = x.Next {
-		chunkView := x.Value
-		if offset < chunkView.ViewOffset {
-			gap := chunkView.ViewOffset - offset
+	for _, chunkView := range chunkViews {
+		if offset < chunkView.LogicOffset {
+			gap := chunkView.LogicOffset - offset
 			remaining -= gap
-			glog.V(4).Infof("zero [%d,%d)", offset, chunkView.ViewOffset)
+			glog.V(4).Infof("zero [%d,%d)", offset, chunkView.LogicOffset)
 			err := writeZero(writer, gap)
 			if err != nil {
-				return fmt.Errorf("write zero [%d,%d)", offset, chunkView.ViewOffset)
+				return fmt.Errorf("write zero [%d,%d)", offset, chunkView.LogicOffset)
 			}
-			offset = chunkView.ViewOffset
+			offset = chunkView.LogicOffset
 		}
 		urlStrings := fileId2Url[chunkView.FileId]
 		start := time.Now()
-		err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize))
-		offset += int64(chunkView.ViewSize)
-		remaining -= int64(chunkView.ViewSize)
+		err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+		offset += int64(chunkView.Size)
+		remaining -= int64(chunkView.Size)
 		stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
 		if err != nil {
 			stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc()
 			return fmt.Errorf("read chunk: %v", err)
 		}
 		stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
-		downloadThrottler.MaybeSlowdown(int64(chunkView.ViewSize))
+		downloadThrottler.MaybeSlowdown(int64(chunkView.Size))
 	}
 	if remaining > 0 {
 		glog.V(4).Infof("zero [%d,%d)", offset, offset+remaining)
@@ -168,15 +167,14 @@ func ReadAll(buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer
 
 	idx := 0
 
-	for x := chunkViews.Front(); x != nil; x = x.Next {
-		chunkView := x.Value
+	for _, chunkView := range chunkViews {
 		urlStrings, err := lookupFileIdFn(chunkView.FileId)
 		if err != nil {
 			glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
 			return err
 		}
 
-		n, err := retriedFetchChunkData(buffer[idx:idx+int(chunkView.ViewSize)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk)
+		n, err := retriedFetchChunkData(buffer[idx:idx+int(chunkView.Size)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset)
 		if err != nil {
 			return err
 		}
@@ -187,8 +185,7 @@ func ReadAll(buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer
 
 // ----------------  ChunkStreamReader ----------------------------------
 type ChunkStreamReader struct {
-	head         *Interval[*ChunkView]
-	chunkView    *Interval[*ChunkView]
+	chunkViews   []*ChunkView
 	totalSize    int64
 	logicOffset  int64
 	buffer       []byte
@@ -204,16 +201,17 @@ var _ = io.ReaderAt(&ChunkStreamReader{})
 func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
 
 	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
+	slices.SortFunc(chunkViews, func(a, b *ChunkView) bool {
+		return a.LogicOffset < b.LogicOffset
+	})
 
 	var totalSize int64
-	for x := chunkViews.Front(); x != nil; x = x.Next {
-		chunk := x.Value
-		totalSize += int64(chunk.ViewSize)
+	for _, chunk := range chunkViews {
+		totalSize += int64(chunk.Size)
 	}
 
 	return &ChunkStreamReader{
-		head:         chunkViews.Front(),
-		chunkView:    chunkViews.Front(),
+		chunkViews:   chunkViews,
 		lookupFileId: lookupFileIdFn,
 		totalSize:    totalSize,
 	}
@@ -292,7 +290,7 @@ func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
 }
 
 func insideChunk(offset int64, chunk *ChunkView) bool {
-	return chunk.ViewOffset <= offset && offset < chunk.ViewOffset+int64(chunk.ViewSize)
+	return chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size)
 }
 
 func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
@@ -302,29 +300,50 @@ func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
 	}
 
 	// fmt.Printf("fetch for offset %d\n", offset)
-	c.chunkView = c.chunkView.Next
-	if c.chunkView == nil {
-		return io.EOF
+
+	// need to seek to a different chunk
+	currentChunkIndex := sort.Search(len(c.chunkViews), func(i int) bool {
+		return offset < c.chunkViews[i].LogicOffset
+	})
+	if currentChunkIndex == len(c.chunkViews) {
+		// not found
+		if insideChunk(offset, c.chunkViews[0]) {
+			// fmt.Printf("select0 chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
+			currentChunkIndex = 0
+		} else if insideChunk(offset, c.chunkViews[len(c.chunkViews)-1]) {
+			currentChunkIndex = len(c.chunkViews) - 1
+			// fmt.Printf("select last chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
+		} else {
+			return io.EOF
+		}
+	} else if currentChunkIndex > 0 {
+		if insideChunk(offset, c.chunkViews[currentChunkIndex]) {
+			// good hit
+		} else if insideChunk(offset, c.chunkViews[currentChunkIndex-1]) {
+			currentChunkIndex -= 1
+			// fmt.Printf("select -1 chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
+		} else {
+			// glog.Fatalf("unexpected1 offset %d", offset)
+			return fmt.Errorf("unexpected1 offset %d", offset)
+		}
+	} else {
+		// glog.Fatalf("unexpected2 offset %d", offset)
+		return fmt.Errorf("unexpected2 offset %d", offset)
 	}
 
 	// positioning within the new chunk
-	chunk := c.chunkView.Value
+	chunk := c.chunkViews[currentChunkIndex]
 	if insideChunk(offset, chunk) {
-		if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
-			return c.fetchChunkToBuffer(chunk)
-		}
-	} else {
-		for p := c.head; p != nil; p = p.Next {
-			chunk = p.Value
-			if insideChunk(offset, chunk) {
-				if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
-					return c.fetchChunkToBuffer(chunk)
-				}
+		if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
+			if err = c.fetchChunkToBuffer(chunk); err != nil {
+				return
 			}
 		}
+	} else {
+		// glog.Fatalf("unexpected3 offset %d in %s [%d,%d)", offset, chunk.FileId, chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
+		return fmt.Errorf("unexpected3 offset %d in %s [%d,%d)", offset, chunk.FileId, chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
 	}
-
-	return io.EOF
+	return
 }
 
 func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
@@ -336,7 +355,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 	var buffer bytes.Buffer
 	var shouldRetry bool
 	for _, urlString := range urlStrings {
-		shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize), func(data []byte) {
+		shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
 			buffer.Write(data)
 		})
 		if !shouldRetry {
@@ -353,10 +372,10 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 		return err
 	}
 	c.buffer = buffer.Bytes()
-	c.bufferOffset = chunkView.ViewOffset
+	c.bufferOffset = chunkView.LogicOffset
 	c.chunk = chunkView.FileId
 
-	// glog.V(0).Infof("fetched %s [%d,%d)", chunkView.FileId, chunkView.ViewOffset, chunkView.ViewOffset+int64(chunkView.ViewSize))
+	// glog.V(0).Infof("fetched %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
 
 	return nil
 }

weed/glog/glog_file.go

@@ -25,7 +25,6 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
-	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -33,7 +32,6 @@ import (
 
 // MaxSize is the maximum size of a log file in bytes.
 var MaxSize uint64 = 1024 * 1024 * 1800
-var MaxFileCount = 5
 
 // logDirs lists the candidate directories for new log files.
 var logDirs []string
@@ -45,9 +43,8 @@ var logDir = flag.String("logdir", "", "If non-empty, write log files in this di
 func createLogDirs() {
 	if *logDir != "" {
 		logDirs = append(logDirs, *logDir)
-	} else {
-		logDirs = append(logDirs, os.TempDir())
 	}
+	logDirs = append(logDirs, os.TempDir())
 }
 
 var (
@@ -99,15 +96,6 @@ func logName(tag string, t time.Time) (name, link string) {
 	return name, program + "." + tag
 }
 
-func prefix(tag string) string {
-	return fmt.Sprintf("%s.%s.%s.log.%s.",
-		program,
-		host,
-		userName,
-		tag,
-	)
-}
-
 var onceLogDirs sync.Once
 
 // create creates a new log file and returns the file and its filename, which
@@ -120,29 +108,8 @@ func create(tag string, t time.Time) (f *os.File, filename string, err error) {
 		return nil, "", errors.New("log: no log dirs")
 	}
 	name, link := logName(tag, t)
-	logPrefix := prefix(tag)
 	var lastErr error
 	for _, dir := range logDirs {
-
-		// remove old logs
-		entries, _ := os.ReadDir(dir)
-		var previousLogs []string
-		for _, entry := range entries {
-			if strings.HasPrefix(entry.Name(), logPrefix) {
-				previousLogs = append(previousLogs, entry.Name())
-			}
-		}
-		if len(previousLogs) >= MaxFileCount {
-			sort.Strings(previousLogs)
-			for i, entry := range previousLogs {
-				if i > len(previousLogs)-MaxFileCount {
-					break
-				}
-				os.Remove(filepath.Join(dir, entry))
-			}
-		}
-
-		// create new log file
 		fname := filepath.Join(dir, name)
 		f, err := os.Create(fname)
 		if err == nil {

weed/iamapi/iamapi_server.go

@@ -77,7 +77,7 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) {
 
 func (iam IamS3ApiConfigure) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) {
 	var buf bytes.Buffer
-	err = pb.WithGrpcFilerClient(false, 0, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		if err = filer.ReadEntry(iam.masterClient, client, filer.IamConfigDirectory, filer.IamIdentityFile, &buf); err != nil {
 			return err
 		}
@@ -99,7 +99,7 @@ func (iam IamS3ApiConfigure) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfigurat
 	if err := filer.ProtoToText(&buf, s3cfg); err != nil {
 		return fmt.Errorf("ProtoToText: %s", err)
 	}
-	return pb.WithGrpcFilerClient(false, 0, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithGrpcFilerClient(false, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		err = util.Retry("saveIamIdentity", func() error {
 			return filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamIdentityFile, buf.Bytes())
 		})
@@ -112,7 +112,7 @@ func (iam IamS3ApiConfigure) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfigurat
 
 func (iam IamS3ApiConfigure) GetPolicies(policies *Policies) (err error) {
 	var buf bytes.Buffer
-	err = pb.WithGrpcFilerClient(false, 0, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		if err = filer.ReadEntry(iam.masterClient, client, filer.IamConfigDirectory, filer.IamPoliciesFile, &buf); err != nil {
 			return err
 		}
@@ -136,7 +136,7 @@ func (iam IamS3ApiConfigure) PutPolicies(policies *Policies) (err error) {
 	if b, err = json.Marshal(policies); err != nil {
 		return err
 	}
-	return pb.WithGrpcFilerClient(false, 0, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithGrpcFilerClient(false, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		if err := filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile, b); err != nil {
 			return err
 		}

weed/images/cropping.go

@@ -1,47 +0,0 @@
-package images
-
-import (
-	"bytes"
-	"image"
-	"image/gif"
-	"image/jpeg"
-	"image/png"
-	"io"
-
-	"github.com/disintegration/imaging"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-)
-
-func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
-	srcImage, _, err := image.Decode(read)
-	if err != nil {
-		glog.Error(err)
-		return read, err
-	}
-
-	bounds := srcImage.Bounds()
-	if x2 > bounds.Dx() || y2 > bounds.Dy() {
-		read.Seek(0, 0)
-		return read, nil
-	}
-
-	rectangle := image.Rect(x1, y1, x2, y2)
-	dstImage := imaging.Crop(srcImage, rectangle)
-	var buf bytes.Buffer
-	switch ext {
-	case ".jpg", ".jpeg":
-		if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
-			glog.Error(err)
-		}
-	case ".png":
-		if err = png.Encode(&buf, dstImage); err != nil {
-			glog.Error(err)
-		}
-	case ".gif":
-		if err = gif.Encode(&buf, dstImage, nil); err != nil {
-			glog.Error(err)
-		}
-	}
-	return bytes.NewReader(buf.Bytes()), err
-}

weed/images/cropping_test.go

@@ -1,22 +0,0 @@
-package images
-
-import (
-	"bytes"
-	"os"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/util"
-)
-
-func TestCropping(t *testing.T) {
-	fname := "sample1.jpg"
-
-	dat, _ := os.ReadFile(fname)
-
-	cropped, _ := Cropped(".jpg", bytes.NewReader(dat), 1072, 932, 1751, 1062)
-	buf := new(bytes.Buffer)
-	buf.ReadFrom(cropped)
-
-	util.WriteFile("cropped1.jpg", buf.Bytes(), 0644)
-
-}

weed/mount/dirty_pages_chunked.go

@@ -7,6 +7,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"io"
 	"sync"
+	"time"
 )
 
 type ChunkedDirtyPages struct {
@@ -37,11 +38,11 @@ func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages {
 	return dirtyPages
 }
 
-func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) {
+func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool) {
 	pages.hasWrites = true
 
 	glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.fh, offset, offset+int64(len(data)))
-	pages.uploadPipeline.SaveDataAt(data, offset, isSequential, tsNs)
+	pages.uploadPipeline.SaveDataAt(data, offset, isSequential)
 
 	return
 }
@@ -57,27 +58,29 @@ func (pages *ChunkedDirtyPages) FlushData() error {
 	return nil
 }
 
-func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64) {
+func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
 	if !pages.hasWrites {
 		return
 	}
-	return pages.uploadPipeline.MaybeReadDataAt(data, startOffset, tsNs)
+	return pages.uploadPipeline.MaybeReadDataAt(data, startOffset)
 }
 
-func (pages *ChunkedDirtyPages) saveChunkedFileIntervalToStorage(reader io.Reader, offset int64, size int64, modifiedTsNs int64, cleanupFn func()) {
+func (pages *ChunkedDirtyPages) saveChunkedFileIntervalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) {
 
+	mtime := time.Now().UnixNano()
 	defer cleanupFn()
 
 	fileFullPath := pages.fh.FullPath()
 	fileName := fileFullPath.Name()
-	chunk, err := pages.fh.wfs.saveDataAsChunk(fileFullPath)(reader, fileName, offset, modifiedTsNs)
+	chunk, err := pages.fh.wfs.saveDataAsChunk(fileFullPath)(reader, fileName, offset)
 	if err != nil {
 		glog.V(0).Infof("%v saveToStorage [%d,%d): %v", fileFullPath, offset, offset+size, err)
 		pages.lastErr = err
 		return
 	}
+	chunk.ModifiedTsNs = mtime
 	pages.fh.AddChunks([]*filer_pb.FileChunk{chunk})
-	pages.fh.entryChunkGroup.AddChunk(chunk)
+	pages.fh.entryViewCache = nil
 	glog.V(3).Infof("%v saveToStorage %s [%d,%d)", fileFullPath, chunk.FileId, offset, offset+size)
 
 }

weed/mount/filehandle.go

@@ -5,60 +5,50 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-	"os"
+	"golang.org/x/exp/slices"
+	"golang.org/x/sync/semaphore"
+	"math"
 	"sync"
 )
 
 type FileHandleId uint64
 
-var IsDebugFileReadWrite = false
-
 type FileHandle struct {
-	fh              FileHandleId
-	counter         int64
-	entry           *LockedEntry
-	entryLock       sync.Mutex
-	entryChunkGroup *filer.ChunkGroup
-	inode           uint64
-	wfs             *WFS
+	fh        FileHandleId
+	counter   int64
+	entry     *LockedEntry
+	entryLock sync.Mutex
+	inode     uint64
+	wfs       *WFS
 
 	// cache file has been written to
-	dirtyMetadata bool
-	dirtyPages    *PageWriter
-	reader        *filer.ChunkReadAt
-	contentType   string
-	handle        uint64
-	sync.Mutex
+	dirtyMetadata  bool
+	dirtyPages     *PageWriter
+	entryViewCache []filer.VisibleInterval
+	reader         *filer.ChunkReadAt
+	contentType    string
+	handle         uint64
+	orderedMutex   *semaphore.Weighted
 
 	isDeleted bool
-
-	// for debugging
-	mirrorFile *os.File
 }
 
 func newFileHandle(wfs *WFS, handleId FileHandleId, inode uint64, entry *filer_pb.Entry) *FileHandle {
 	fh := &FileHandle{
-		fh:      handleId,
-		counter: 1,
-		inode:   inode,
-		wfs:     wfs,
+		fh:           handleId,
+		counter:      1,
+		inode:        inode,
+		wfs:          wfs,
+		orderedMutex: semaphore.NewWeighted(int64(math.MaxInt64)),
 	}
 	// dirtyPages: newContinuousDirtyPages(file, writeOnly),
 	fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit)
+	if entry != nil {
+		entry.Attributes.FileSize = filer.FileSize(entry)
+	}
 	fh.entry = &LockedEntry{
 		Entry: entry,
 	}
-	if entry != nil {
-		fh.SetEntry(entry)
-	}
-
-	if IsDebugFileReadWrite {
-		var err error
-		fh.mirrorFile, err = os.OpenFile("/tmp/sw/"+entry.Name, os.O_RDWR|os.O_CREATE, 0600)
-		if err != nil {
-			println("failed to create mirror:", err.Error())
-		}
-	}
 
 	return fh
 }
@@ -73,17 +63,6 @@ func (fh *FileHandle) GetEntry() *filer_pb.Entry {
 }
 
 func (fh *FileHandle) SetEntry(entry *filer_pb.Entry) {
-	if entry != nil {
-		fileSize := filer.FileSize(entry)
-		entry.Attributes.FileSize = fileSize
-		var resolveManifestErr error
-		fh.entryChunkGroup, resolveManifestErr = filer.NewChunkGroup(fh.wfs.LookupFn(), fh.wfs.chunkCache, entry.Chunks)
-		if resolveManifestErr != nil {
-			glog.Warningf("failed to resolve manifest chunks in %+v", entry)
-		}
-	} else {
-		glog.Fatalf("setting file handle entry to nil")
-	}
 	fh.entry.SetEntry(entry)
 }
 
@@ -92,26 +71,52 @@ func (fh *FileHandle) UpdateEntry(fn func(entry *filer_pb.Entry)) *filer_pb.Entr
 }
 
 func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
-	fh.entryLock.Lock()
-	defer fh.entryLock.Unlock()
+	//fh.entryLock.Lock()
+	//defer fh.entryLock.Unlock()
 
 	if fh.entry == nil {
 		return
 	}
 
-	fh.entry.AppendChunks(chunks)
+	// find the earliest incoming chunk
+	newChunks := chunks
+	earliestChunk := newChunks[0]
+	for i := 1; i < len(newChunks); i++ {
+		if lessThan(earliestChunk, newChunks[i]) {
+			earliestChunk = newChunks[i]
+		}
+	}
+
+	// pick out-of-order chunks from existing chunks
+	for _, chunk := range fh.entry.GetChunks() {
+		if lessThan(earliestChunk, chunk) {
+			chunks = append(chunks, chunk)
+		}
+	}
+
+	// sort incoming chunks
+	slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
+		return lessThan(a, b)
+	})
+
+	glog.V(4).Infof("%s existing %d chunks adds %d more", fh.FullPath(), len(fh.entry.GetChunks()), len(chunks))
+
+	fh.entry.AppendChunks(newChunks)
+	fh.entryViewCache = nil
 }
 
-func (fh *FileHandle) ReleaseHandle() {
-	fh.entryLock.Lock()
-	defer fh.entryLock.Unlock()
-
-	fh.dirtyPages.Destroy()
-	if IsDebugFileReadWrite {
-		fh.mirrorFile.Close()
+func (fh *FileHandle) CloseReader() {
+	if fh.reader != nil {
+		_ = fh.reader.Close()
+		fh.reader = nil
 	}
 }
 
+func (fh *FileHandle) Release() {
+	fh.dirtyPages.Destroy()
+	fh.CloseReader()
+}
+
 func lessThan(a, b *filer_pb.FileChunk) bool {
 	if a.ModifiedTsNs == b.ModifiedTsNs {
 		return a.Fid.FileKey < b.Fid.FileKey

weed/mount/filehandle_map.go

@@ -65,7 +65,7 @@ func (i *FileHandleToInode) ReleaseByInode(inode uint64) {
 		if fh.counter <= 0 {
 			delete(i.inode2fh, inode)
 			delete(i.fh2inode, fh.fh)
-			fh.ReleaseHandle()
+			fh.Release()
 		}
 	}
 }
@@ -82,7 +82,7 @@ func (i *FileHandleToInode) ReleaseByHandle(fh FileHandleId) {
 			if fhHandle.counter <= 0 {
 				delete(i.inode2fh, inode)
 				delete(i.fh2inode, fhHandle.fh)
-				fhHandle.ReleaseHandle()
+				fhHandle.Release()
 			}
 		}
 

weed/mount/filehandle_read.go

@@ -17,20 +17,18 @@ func (fh *FileHandle) unlockForRead(startOffset int64, size int) {
 	fh.dirtyPages.UnlockForRead(startOffset, startOffset+int64(size))
 }
 
-func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64, tsNs int64) (maxStop int64) {
-	maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset, tsNs)
+func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxStop int64) {
+	maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset)
 	return
 }
 
-func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, error) {
-	fh.entryLock.Lock()
-	defer fh.entryLock.Unlock()
+func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 
 	fileFullPath := fh.FullPath()
 
 	entry := fh.GetEntry()
 	if entry == nil {
-		return 0, 0, io.EOF
+		return 0, io.EOF
 	}
 
 	if entry.IsInRemoteOnly() {
@@ -38,28 +36,43 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e
 		newEntry, err := fh.downloadRemoteEntry(entry)
 		if err != nil {
 			glog.V(1).Infof("download remote entry %s: %v", fileFullPath, err)
-			return 0, 0, err
+			return 0, err
 		}
 		entry = newEntry
 	}
 
-	fileSize := int64(entry.Attributes.FileSize)
-	if fileSize == 0 {
-		fileSize = int64(filer.FileSize(entry))
-	}
+	fileSize := int64(filer.FileSize(entry))
 
 	if fileSize == 0 {
 		glog.V(1).Infof("empty fh %v", fileFullPath)
-		return 0, 0, io.EOF
+		return 0, io.EOF
 	}
 
 	if offset+int64(len(buff)) <= int64(len(entry.Content)) {
 		totalRead := copy(buff, entry.Content[offset:])
 		glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
-		return int64(totalRead), 0, nil
+		return int64(totalRead), nil
 	}
 
-	totalRead, ts, err := fh.entryChunkGroup.ReadDataAt(fileSize, buff, offset)
+	var chunkResolveErr error
+	if fh.entryViewCache == nil {
+		fh.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.wfs.LookupFn(), entry.GetChunks(), 0, fileSize)
+		if chunkResolveErr != nil {
+			return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
+		}
+		fh.CloseReader()
+	}
+
+	if fh.reader == nil {
+		chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, 0, fileSize)
+		glog.V(4).Infof("file handle read %s [%d,%d) from %d views", fileFullPath, offset, offset+int64(len(buff)), len(chunkViews))
+		for _, chunkView := range chunkViews {
+			glog.V(4).Infof("  read %s [%d,%d) from chunk %+v", fileFullPath, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.FileId)
+		}
+		fh.reader = filer.NewChunkReaderAtFromClient(fh.wfs.LookupFn(), chunkViews, fh.wfs.chunkCache, fileSize)
+	}
+
+	totalRead, err := fh.reader.ReadAt(buff, offset)
 
 	if err != nil && err != io.EOF {
 		glog.Errorf("file handle read %s: %v", fileFullPath, err)
@@ -67,7 +80,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e
 
 	// glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fileFullPath, offset, offset+int64(totalRead), totalRead, err)
 
-	return int64(totalRead), ts, err
+	return int64(totalRead), err
 }
 
 func (fh *FileHandle) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {

weed/mount/meta_cache/meta_cache.go

@@ -2,13 +2,12 @@ package meta_cache
 
 import (
 	"context"
-	"os"
-
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/filer/leveldb"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"os"
 )
 
 // need to have logic similar to FilerStoreWrapper
@@ -70,18 +69,13 @@ func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath uti
 	//mc.Lock()
 	//defer mc.Unlock()
 
-	entry, err := mc.FindEntry(ctx, oldPath)
-	if err != nil && err != filer_pb.ErrNotFound {
-		glog.Errorf("Metacache: find entry error: %v", err)
-		return err
-	}
-	if entry != nil {
+	oldDir, _ := oldPath.DirAndName()
+	if mc.isCachedFn(util.FullPath(oldDir)) {
 		if oldPath != "" {
 			if newEntry != nil && oldPath == newEntry.FullPath {
 				// skip the unnecessary deletion
 				// leave the update to the following InsertEntry operation
 			} else {
-				ctx = context.WithValue(ctx, "OP", "MV")
 				glog.V(3).Infof("DeleteEntry %s", oldPath)
 				if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil {
 					return err

weed/mount/page_writer.go

@@ -29,35 +29,35 @@ func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter {
 	return pw
 }
 
-func (pw *PageWriter) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) {
+func (pw *PageWriter) AddPage(offset int64, data []byte, isSequential bool) {
 
 	glog.V(4).Infof("%v AddPage [%d, %d)", pw.fh.fh, offset, offset+int64(len(data)))
 
 	chunkIndex := offset / pw.chunkSize
 	for i := chunkIndex; len(data) > 0; i++ {
 		writeSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
-		pw.addToOneChunk(i, offset, data[:writeSize], isSequential, tsNs)
+		pw.addToOneChunk(i, offset, data[:writeSize], isSequential)
 		offset += writeSize
 		data = data[writeSize:]
 	}
 }
 
-func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte, isSequential bool, tsNs int64) {
-	pw.randomWriter.AddPage(offset, data, isSequential, tsNs)
+func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte, isSequential bool) {
+	pw.randomWriter.AddPage(offset, data, isSequential)
 }
 
 func (pw *PageWriter) FlushData() error {
 	return pw.randomWriter.FlushData()
 }
 
-func (pw *PageWriter) ReadDirtyDataAt(data []byte, offset int64, tsNs int64) (maxStop int64) {
+func (pw *PageWriter) ReadDirtyDataAt(data []byte, offset int64) (maxStop int64) {
 	glog.V(4).Infof("ReadDirtyDataAt %v [%d, %d)", pw.fh.fh, offset, offset+int64(len(data)))
 
 	chunkIndex := offset / pw.chunkSize
 	for i := chunkIndex; len(data) > 0; i++ {
 		readSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
 
-		maxStop = pw.randomWriter.ReadDirtyDataAt(data[:readSize], offset, tsNs)
+		maxStop = pw.randomWriter.ReadDirtyDataAt(data[:readSize], offset)
 
 		offset += readSize
 		data = data[readSize:]

weed/mount/page_writer/activity_score.go

@@ -1,39 +0,0 @@
-package page_writer
-
-import "time"
-
-type ActivityScore struct {
-	lastActiveTsNs         int64
-	decayedActivenessScore int64
-}
-
-func NewActivityScore() *ActivityScore {
-	return &ActivityScore{}
-}
-
-func (as ActivityScore) MarkRead() {
-	now := time.Now().UnixNano()
-	deltaTime := (now - as.lastActiveTsNs) >> 30 // about number of seconds
-	as.lastActiveTsNs = now
-
-	as.decayedActivenessScore = as.decayedActivenessScore>>deltaTime + 256
-	if as.decayedActivenessScore < 0 {
-		as.decayedActivenessScore = 0
-	}
-}
-
-func (as ActivityScore) MarkWrite() {
-	now := time.Now().UnixNano()
-	deltaTime := (now - as.lastActiveTsNs) >> 30 // about number of seconds
-	as.lastActiveTsNs = now
-
-	as.decayedActivenessScore = as.decayedActivenessScore>>deltaTime + 1024
-	if as.decayedActivenessScore < 0 {
-		as.decayedActivenessScore = 0
-	}
-}
-
-func (as ActivityScore) ActivityScore() int64 {
-	deltaTime := (time.Now().UnixNano() - as.lastActiveTsNs) >> 30 // about number of seconds
-	return as.decayedActivenessScore >> deltaTime
-}

weed/mount/page_writer/chunk_interval_list.go

@@ -8,7 +8,6 @@ import (
 type ChunkWrittenInterval struct {
 	StartOffset int64
 	stopOffset  int64
-	TsNs        int64
 	prev        *ChunkWrittenInterval
 	next        *ChunkWrittenInterval
 }
@@ -43,14 +42,10 @@ func newChunkWrittenIntervalList() *ChunkWrittenIntervalList {
 	return list
 }
 
-func (list *ChunkWrittenIntervalList) MarkWritten(startOffset, stopOffset, tsNs int64) {
-	if startOffset >= stopOffset {
-		return
-	}
+func (list *ChunkWrittenIntervalList) MarkWritten(startOffset, stopOffset int64) {
 	interval := &ChunkWrittenInterval{
 		StartOffset: startOffset,
 		stopOffset:  stopOffset,
-		TsNs:        tsNs,
 	}
 	list.addInterval(interval)
 }
@@ -67,56 +62,52 @@ func (list *ChunkWrittenIntervalList) WrittenSize() (writtenByteCount int64) {
 
 func (list *ChunkWrittenIntervalList) addInterval(interval *ChunkWrittenInterval) {
 
-	//t := list.head
-	//for ; t.next != nil; t = t.next {
-	//	if t.TsNs > interval.TsNs {
-	//		println("writes is out of order", t.TsNs-interval.TsNs, "ns")
-	//	}
-	//}
-
 	p := list.head
-	for ; p.next != nil && p.next.stopOffset <= interval.StartOffset; p = p.next {
+	for ; p.next != nil && p.next.StartOffset <= interval.StartOffset; p = p.next {
 	}
 	q := list.tail
-	for ; q.prev != nil && q.prev.StartOffset >= interval.stopOffset; q = q.prev {
+	for ; q.prev != nil && q.prev.stopOffset >= interval.stopOffset; q = q.prev {
 	}
 
-	// left side
-	// interval after p.next start
-	if p.next.StartOffset < interval.StartOffset {
-		t := &ChunkWrittenInterval{
-			StartOffset: p.next.StartOffset,
-			stopOffset:  interval.StartOffset,
-			TsNs:        p.next.TsNs,
-		}
-		p.next = t
-		t.prev = p
-		t.next = interval
-		interval.prev = t
-	} else {
-		p.next = interval
-		interval.prev = p
+	if interval.StartOffset <= p.stopOffset && q.StartOffset <= interval.stopOffset {
+		// merge p and q together
+		p.stopOffset = q.stopOffset
+		unlinkNodesBetween(p, q.next)
+		return
+	}
+	if interval.StartOffset <= p.stopOffset {
+		// merge new interval into p
+		p.stopOffset = interval.stopOffset
+		unlinkNodesBetween(p, q)
+		return
+	}
+	if q.StartOffset <= interval.stopOffset {
+		// merge new interval into q
+		q.StartOffset = interval.StartOffset
+		unlinkNodesBetween(p, q)
+		return
 	}
 
-	// right side
-	// interval ends before p.prev
-	if interval.stopOffset < q.prev.stopOffset {
-		t := &ChunkWrittenInterval{
-			StartOffset: interval.stopOffset,
-			stopOffset:  q.prev.stopOffset,
-			TsNs:        q.prev.TsNs,
-		}
-		q.prev = t
-		t.next = q
-		interval.next = t
-		t.prev = interval
-	} else {
-		q.prev = interval
-		interval.next = q
-	}
+	// add the new interval between p and q
+	unlinkNodesBetween(p, q)
+	p.next = interval
+	interval.prev = p
+	q.prev = interval
+	interval.next = q
 
 }
 
+// unlinkNodesBetween remove all nodes after start and before stop, exclusive
+func unlinkNodesBetween(start *ChunkWrittenInterval, stop *ChunkWrittenInterval) {
+	if start.next == stop {
+		return
+	}
+	start.next.prev = nil
+	start.next = stop
+	stop.prev.next = nil
+	stop.prev = start
+}
+
 func (list *ChunkWrittenIntervalList) size() int {
 	var count int
 	for t := list.head; t != nil; t = t.next {

weed/mount/page_writer/chunk_interval_list_test.go

@@ -10,72 +10,40 @@ func Test_PageChunkWrittenIntervalList(t *testing.T) {
 
 	assert.Equal(t, 0, list.size(), "empty list")
 
-	list.MarkWritten(0, 5, 1)
+	list.MarkWritten(0, 5)
 	assert.Equal(t, 1, list.size(), "one interval")
 
-	list.MarkWritten(0, 5, 2)
+	list.MarkWritten(0, 5)
 	assert.Equal(t, 1, list.size(), "duplicated interval2")
 
-	list.MarkWritten(95, 100, 3)
+	list.MarkWritten(95, 100)
 	assert.Equal(t, 2, list.size(), "two intervals")
 
-	list.MarkWritten(50, 60, 4)
+	list.MarkWritten(50, 60)
 	assert.Equal(t, 3, list.size(), "three intervals")
 
-	list.MarkWritten(50, 55, 5)
-	assert.Equal(t, 4, list.size(), "three intervals merge")
+	list.MarkWritten(50, 55)
+	assert.Equal(t, 3, list.size(), "three intervals merge")
 
-	list.MarkWritten(40, 50, 6)
-	assert.Equal(t, 5, list.size(), "three intervals grow forward")
+	list.MarkWritten(40, 50)
+	assert.Equal(t, 3, list.size(), "three intervals grow forward")
 
-	list.MarkWritten(50, 65, 7)
-	assert.Equal(t, 4, list.size(), "three intervals grow backward")
+	list.MarkWritten(50, 65)
+	assert.Equal(t, 3, list.size(), "three intervals grow backward")
 
-	list.MarkWritten(70, 80, 8)
-	assert.Equal(t, 5, list.size(), "four intervals")
+	list.MarkWritten(70, 80)
+	assert.Equal(t, 4, list.size(), "four intervals")
 
-	list.MarkWritten(60, 70, 9)
-	assert.Equal(t, 6, list.size(), "three intervals merged")
+	list.MarkWritten(60, 70)
+	assert.Equal(t, 3, list.size(), "three intervals merged")
 
-	list.MarkWritten(59, 71, 10)
-	assert.Equal(t, 6, list.size(), "covered three intervals")
+	list.MarkWritten(59, 71)
+	assert.Equal(t, 3, list.size(), "covered three intervals")
 
-	list.MarkWritten(5, 59, 11)
-	assert.Equal(t, 5, list.size(), "covered two intervals")
+	list.MarkWritten(5, 59)
+	assert.Equal(t, 2, list.size(), "covered two intervals")
 
-	list.MarkWritten(70, 99, 12)
-	assert.Equal(t, 5, list.size(), "covered one intervals")
+	list.MarkWritten(70, 99)
+	assert.Equal(t, 1, list.size(), "covered one intervals")
 
 }
-
-type interval struct {
-	start    int64
-	stop     int64
-	expected bool
-}
-
-func Test_PageChunkWrittenIntervalList1(t *testing.T) {
-	list := newChunkWrittenIntervalList()
-	inputs := []interval{
-		{1, 5, true},
-		{2, 3, true},
-	}
-	for i, input := range inputs {
-		list.MarkWritten(input.start, input.stop, int64(i)+1)
-		actual := hasData(list, 0, 4)
-		if actual != input.expected {
-			t.Errorf("input [%d,%d) expected %v actual %v", input.start, input.stop, input.expected, actual)
-		}
-	}
-}
-
-func hasData(usage *ChunkWrittenIntervalList, chunkStartOffset, x int64) bool {
-	for t := usage.head.next; t != usage.tail; t = t.next {
-		logicStart := chunkStartOffset + t.StartOffset
-		logicStop := chunkStartOffset + t.stopOffset
-		if logicStart <= x && x < logicStop {
-			return true
-		}
-	}
-	return false
-}

weed/mount/page_writer/dirty_pages.go

@@ -1,9 +1,9 @@
 package page_writer
 
 type DirtyPages interface {
-	AddPage(offset int64, data []byte, isSequential bool, tsNs int64)
+	AddPage(offset int64, data []byte, isSequential bool)
 	FlushData() error
-	ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64)
+	ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64)
 	Destroy()
 	LockForRead(startOffset, stopOffset int64)
 	UnlockForRead(startOffset, stopOffset int64)

weed/mount/page_writer/page_chunk.go

@@ -4,14 +4,13 @@ import (
 	"io"
 )
 
-type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, modifiedTsNs int64, cleanupFn func())
+type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, cleanupFn func())
 
 type PageChunk interface {
 	FreeResource()
-	WriteDataAt(src []byte, offset int64, tsNs int64) (n int)
-	ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64)
+	WriteDataAt(src []byte, offset int64) (n int)
+	ReadDataAt(p []byte, off int64) (maxStop int64)
 	IsComplete() bool
-	ActivityScore() int64
 	WrittenSize() int64
 	SaveContent(saveFn SaveToStorageFunc)
 }

weed/mount/page_writer/page_chunk_mem.go

@@ -19,7 +19,6 @@ type MemChunk struct {
 	usage           *ChunkWrittenIntervalList
 	chunkSize       int64
 	logicChunkIndex LogicChunkIndex
-	activityScore   *ActivityScore
 }
 
 func NewMemChunk(logicChunkIndex LogicChunkIndex, chunkSize int64) *MemChunk {
@@ -29,7 +28,6 @@ func NewMemChunk(logicChunkIndex LogicChunkIndex, chunkSize int64) *MemChunk {
 		chunkSize:       chunkSize,
 		buf:             mem.Allocate(int(chunkSize)),
 		usage:           newChunkWrittenIntervalList(),
-		activityScore:   NewActivityScore(),
 	}
 }
 
@@ -41,37 +39,29 @@ func (mc *MemChunk) FreeResource() {
 	mem.Free(mc.buf)
 }
 
-func (mc *MemChunk) WriteDataAt(src []byte, offset int64, tsNs int64) (n int) {
+func (mc *MemChunk) WriteDataAt(src []byte, offset int64) (n int) {
 	mc.Lock()
 	defer mc.Unlock()
 
 	innerOffset := offset % mc.chunkSize
 	n = copy(mc.buf[innerOffset:], src)
-	mc.usage.MarkWritten(innerOffset, innerOffset+int64(n), tsNs)
-	mc.activityScore.MarkWrite()
-
+	mc.usage.MarkWritten(innerOffset, innerOffset+int64(n))
 	return
 }
 
-func (mc *MemChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+func (mc *MemChunk) ReadDataAt(p []byte, off int64) (maxStop int64) {
 	mc.RLock()
 	defer mc.RUnlock()
 
 	memChunkBaseOffset := int64(mc.logicChunkIndex) * mc.chunkSize
 	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
-		logicStart := max(off, memChunkBaseOffset+t.StartOffset)
+		logicStart := max(off, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset)
 		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
 		if logicStart < logicStop {
-			if t.TsNs >= tsNs {
-				copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
-				maxStop = max(maxStop, logicStop)
-			} else {
-				println("read old data1", tsNs-t.TsNs, "ns")
-			}
+			copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
+			maxStop = max(maxStop, logicStop)
 		}
 	}
-	mc.activityScore.MarkRead()
-
 	return
 }
 
@@ -82,10 +72,6 @@ func (mc *MemChunk) IsComplete() bool {
 	return mc.usage.IsComplete(mc.chunkSize)
 }
 
-func (mc *MemChunk) ActivityScore() int64 {
-	return mc.activityScore.ActivityScore()
-}
-
 func (mc *MemChunk) WrittenSize() int64 {
 	mc.RLock()
 	defer mc.RUnlock()
@@ -102,21 +88,7 @@ func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) {
 	}
 	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
 		reader := util.NewBytesReader(mc.buf[t.StartOffset:t.stopOffset])
-		saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), t.TsNs, func() {
-		})
-	}
-
-	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
-		startOffset := t.StartOffset
-		stopOffset := t.stopOffset
-		tsNs := t.TsNs
-		for t != mc.usage.tail && t.next.StartOffset == stopOffset {
-			stopOffset = t.next.stopOffset
-			t = t.next
-			tsNs = max(tsNs, t.TsNs)
-		}
-		reader := util.NewBytesReader(mc.buf[startOffset:stopOffset])
-		saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+startOffset, stopOffset-startOffset, tsNs, func() {
+		saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), func() {
 		})
 	}
 }

weed/mount/page_writer/page_chunk_swapfile.go

@@ -15,12 +15,12 @@ var (
 type ActualChunkIndex int
 
 type SwapFile struct {
-	dir                 string
-	file                *os.File
-	chunkSize           int64
-	chunkTrackingLock   sync.Mutex
-	activeChunkCount    int
-	freeActualChunkList []ActualChunkIndex
+	dir                         string
+	file                        *os.File
+	logicToActualChunkIndex     map[LogicChunkIndex]ActualChunkIndex
+	logicToActualChunkIndexLock sync.Mutex
+	chunkSize                   int64
+	freeActualChunkList         []ActualChunkIndex
 }
 
 type SwapFileChunk struct {
@@ -29,15 +29,14 @@ type SwapFileChunk struct {
 	usage            *ChunkWrittenIntervalList
 	logicChunkIndex  LogicChunkIndex
 	actualChunkIndex ActualChunkIndex
-	activityScore    *ActivityScore
-	//memChunk         *MemChunk
 }
 
 func NewSwapFile(dir string, chunkSize int64) *SwapFile {
 	return &SwapFile{
-		dir:       dir,
-		file:      nil,
-		chunkSize: chunkSize,
+		dir:                     dir,
+		file:                    nil,
+		logicToActualChunkIndex: make(map[LogicChunkIndex]ActualChunkIndex),
+		chunkSize:               chunkSize,
 	}
 }
 func (sf *SwapFile) FreeResource() {
@@ -47,7 +46,7 @@ func (sf *SwapFile) FreeResource() {
 	}
 }
 
-func (sf *SwapFile) NewSwapFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) {
+func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) {
 	if sf.file == nil {
 		var err error
 		sf.file, err = os.CreateTemp(sf.dir, "")
@@ -56,98 +55,70 @@ func (sf *SwapFile) NewSwapFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapF
 			return nil
 		}
 	}
-	sf.chunkTrackingLock.Lock()
-	defer sf.chunkTrackingLock.Unlock()
-
-	sf.activeChunkCount++
-
-	// assign a new physical chunk
-	var actualChunkIndex ActualChunkIndex
-	if len(sf.freeActualChunkList) > 0 {
-		actualChunkIndex = sf.freeActualChunkList[0]
-		sf.freeActualChunkList = sf.freeActualChunkList[1:]
-	} else {
-		actualChunkIndex = ActualChunkIndex(sf.activeChunkCount)
+	sf.logicToActualChunkIndexLock.Lock()
+	defer sf.logicToActualChunkIndexLock.Unlock()
+	actualChunkIndex, found := sf.logicToActualChunkIndex[logicChunkIndex]
+	if !found {
+		if len(sf.freeActualChunkList) > 0 {
+			actualChunkIndex = sf.freeActualChunkList[0]
+			sf.freeActualChunkList = sf.freeActualChunkList[1:]
+		} else {
+			actualChunkIndex = ActualChunkIndex(len(sf.logicToActualChunkIndex))
+		}
+		sf.logicToActualChunkIndex[logicChunkIndex] = actualChunkIndex
 	}
 
-	swapFileChunk := &SwapFileChunk{
+	return &SwapFileChunk{
 		swapfile:         sf,
 		usage:            newChunkWrittenIntervalList(),
 		logicChunkIndex:  logicChunkIndex,
 		actualChunkIndex: actualChunkIndex,
-		activityScore:    NewActivityScore(),
-		// memChunk:         NewMemChunk(logicChunkIndex, sf.chunkSize),
 	}
-
-	// println(logicChunkIndex, "|", "++++", swapFileChunk.actualChunkIndex, swapFileChunk, sf)
-	return swapFileChunk
 }
 
 func (sc *SwapFileChunk) FreeResource() {
+	sc.swapfile.logicToActualChunkIndexLock.Lock()
+	defer sc.swapfile.logicToActualChunkIndexLock.Unlock()
 
 	sc.Lock()
 	defer sc.Unlock()
 
-	sc.swapfile.chunkTrackingLock.Lock()
-	defer sc.swapfile.chunkTrackingLock.Unlock()
-
 	sc.swapfile.freeActualChunkList = append(sc.swapfile.freeActualChunkList, sc.actualChunkIndex)
-	sc.swapfile.activeChunkCount--
-	// println(sc.logicChunkIndex, "|", "----", sc.actualChunkIndex, sc, sc.swapfile)
+	delete(sc.swapfile.logicToActualChunkIndex, sc.logicChunkIndex)
 }
 
-func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64, tsNs int64) (n int) {
+func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64) (n int) {
 	sc.Lock()
 	defer sc.Unlock()
 
-	// println(sc.logicChunkIndex, "|", tsNs, "write at", offset, len(src), sc.actualChunkIndex)
-
 	innerOffset := offset % sc.swapfile.chunkSize
 	var err error
 	n, err = sc.swapfile.file.WriteAt(src, int64(sc.actualChunkIndex)*sc.swapfile.chunkSize+innerOffset)
-	sc.usage.MarkWritten(innerOffset, innerOffset+int64(n), tsNs)
-	if err != nil {
+	if err == nil {
+		sc.usage.MarkWritten(innerOffset, innerOffset+int64(n))
+	} else {
 		glog.Errorf("failed to write swap file %s: %v", sc.swapfile.file.Name(), err)
 	}
-	//sc.memChunk.WriteDataAt(src, offset, tsNs)
-	sc.activityScore.MarkWrite()
-
 	return
 }
 
-func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64) (maxStop int64) {
 	sc.RLock()
 	defer sc.RUnlock()
 
-	// println(sc.logicChunkIndex, "|", tsNs, "read at", off, len(p), sc.actualChunkIndex)
-
-	//memCopy := make([]byte, len(p))
-	//copy(memCopy, p)
-
 	chunkStartOffset := int64(sc.logicChunkIndex) * sc.swapfile.chunkSize
 	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
 		logicStart := max(off, chunkStartOffset+t.StartOffset)
 		logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset)
 		if logicStart < logicStop {
-			if t.TsNs >= tsNs {
-				actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
-				if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
-					glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
-					break
-				}
-				maxStop = max(maxStop, logicStop)
-			} else {
-				println("read old data2", tsNs-t.TsNs, "ns")
+			actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
+			if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
+				glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
+				break
 			}
+			maxStop = max(maxStop, logicStop)
 		}
 	}
-	//sc.memChunk.ReadDataAt(memCopy, off, tsNs)
-	//if bytes.Compare(memCopy, p) != 0 {
-	//	println("read wrong data from swap file", off, sc.logicChunkIndex)
-	//}
-
-	sc.activityScore.MarkRead()
-
 	return
 }
 
@@ -157,10 +128,6 @@ func (sc *SwapFileChunk) IsComplete() bool {
 	return sc.usage.IsComplete(sc.swapfile.chunkSize)
 }
 
-func (sc *SwapFileChunk) ActivityScore() int64 {
-	return sc.activityScore.ActivityScore()
-}
-
 func (sc *SwapFileChunk) WrittenSize() int64 {
 	sc.RLock()
 	defer sc.RUnlock()
@@ -168,31 +135,20 @@ func (sc *SwapFileChunk) WrittenSize() int64 {
 }
 
 func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) {
-	sc.RLock()
-	defer sc.RUnlock()
-
 	if saveFn == nil {
 		return
 	}
-	// println(sc.logicChunkIndex, "|", "save")
-	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
-		startOffset := t.StartOffset
-		stopOffset := t.stopOffset
-		tsNs := t.TsNs
-		for t != sc.usage.tail && t.next.StartOffset == stopOffset {
-			stopOffset = t.next.stopOffset
-			t = t.next
-			tsNs = max(tsNs, t.TsNs)
-		}
+	sc.Lock()
+	defer sc.Unlock()
 
-		data := mem.Allocate(int(stopOffset - startOffset))
-		n, _ := sc.swapfile.file.ReadAt(data, startOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
-		if n > 0 {
-			reader := util.NewBytesReader(data[:n])
-			saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+startOffset, int64(n), tsNs, func() {
-			})
-		}
+	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
+		data := mem.Allocate(int(t.Size()))
+		sc.swapfile.file.ReadAt(data, t.StartOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
+		reader := util.NewBytesReader(data)
+		saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+t.StartOffset, t.Size(), func() {
+		})
 		mem.Free(data)
 	}
 
+	sc.usage = newChunkWrittenIntervalList()
 }

weed/mount/page_writer/upload_pipeline.go

@@ -6,24 +6,26 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sync"
 	"sync/atomic"
+	"time"
 )
 
 type LogicChunkIndex int
 
 type UploadPipeline struct {
-	uploaderCount      int32
-	uploaderCountCond  *sync.Cond
-	filepath           util.FullPath
-	ChunkSize          int64
-	uploaders          *util.LimitedConcurrentExecutor
-	saveToStorageFn    SaveToStorageFunc
-	writableChunkLimit int
-	swapFile           *SwapFile
-	chunksLock         sync.Mutex
-	writableChunks     map[LogicChunkIndex]PageChunk
-	sealedChunks       map[LogicChunkIndex]*SealedChunk
-	activeReadChunks   map[LogicChunkIndex]int
-	readerCountCond    *sync.Cond
+	uploaderCount        int32
+	uploaderCountCond    *sync.Cond
+	filepath             util.FullPath
+	ChunkSize            int64
+	writableChunks       map[LogicChunkIndex]PageChunk
+	writableChunksLock   sync.Mutex
+	sealedChunks         map[LogicChunkIndex]*SealedChunk
+	sealedChunksLock     sync.Mutex
+	uploaders            *util.LimitedConcurrentExecutor
+	saveToStorageFn      SaveToStorageFunc
+	activeReadChunks     map[LogicChunkIndex]int
+	activeReadChunksLock sync.Mutex
+	writableChunkLimit   int
+	swapFile             *SwapFile
 }
 
 type SealedChunk struct {
@@ -40,7 +42,7 @@ func (sc *SealedChunk) FreeReference(messageOnFree string) {
 }
 
 func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int, swapFileDir string) *UploadPipeline {
-	t := &UploadPipeline{
+	return &UploadPipeline{
 		ChunkSize:          chunkSize,
 		writableChunks:     make(map[LogicChunkIndex]PageChunk),
 		sealedChunks:       make(map[LogicChunkIndex]*SealedChunk),
@@ -51,14 +53,11 @@ func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64,
 		writableChunkLimit: bufferChunkLimit,
 		swapFile:           NewSwapFile(swapFileDir, chunkSize),
 	}
-	t.readerCountCond = sync.NewCond(&t.chunksLock)
-	return t
 }
 
-func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsNs int64) (n int) {
-
-	up.chunksLock.Lock()
-	defer up.chunksLock.Unlock()
+func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n int) {
+	up.writableChunksLock.Lock()
+	defer up.writableChunksLock.Unlock()
 
 	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
 
@@ -66,80 +65,57 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsN
 	if !found {
 		if len(up.writableChunks) > up.writableChunkLimit {
 			// if current file chunks is over the per file buffer count limit
-			candidateChunkIndex, fullness := LogicChunkIndex(-1), int64(0)
+			fullestChunkIndex, fullness := LogicChunkIndex(-1), int64(0)
 			for lci, mc := range up.writableChunks {
 				chunkFullness := mc.WrittenSize()
 				if fullness < chunkFullness {
-					candidateChunkIndex = lci
+					fullestChunkIndex = lci
 					fullness = chunkFullness
 				}
 			}
-			/*  // this algo generates too many chunks
-			candidateChunkIndex, lowestActivityScore := LogicChunkIndex(-1), int64(math.MaxInt64)
-			for wci, wc := range up.writableChunks {
-				activityScore := wc.ActivityScore()
-				if lowestActivityScore >= activityScore {
-					if lowestActivityScore == activityScore {
-						chunkFullness := wc.WrittenSize()
-						if fullness < chunkFullness {
-							candidateChunkIndex = lci
-							fullness = chunkFullness
-						}
-					}
-					candidateChunkIndex = wci
-					lowestActivityScore = activityScore
-				}
-			}
-			*/
-			up.moveToSealed(up.writableChunks[candidateChunkIndex], candidateChunkIndex)
+			up.moveToSealed(up.writableChunks[fullestChunkIndex], fullestChunkIndex)
+			delete(up.writableChunks, fullestChunkIndex)
 			// fmt.Printf("flush chunk %d with %d bytes written\n", logicChunkIndex, fullness)
 		}
-		// fmt.Printf("isSequential:%v len(up.writableChunks):%v memChunkCounter:%v", isSequential, len(up.writableChunks), memChunkCounter)
 		if isSequential &&
 			len(up.writableChunks) < up.writableChunkLimit &&
 			atomic.LoadInt64(&memChunkCounter) < 4*int64(up.writableChunkLimit) {
 			pageChunk = NewMemChunk(logicChunkIndex, up.ChunkSize)
-			// fmt.Printf(" create mem  chunk %d\n", logicChunkIndex)
 		} else {
-			pageChunk = up.swapFile.NewSwapFileChunk(logicChunkIndex)
-			// fmt.Printf(" create file chunk %d\n", logicChunkIndex)
+			pageChunk = up.swapFile.NewTempFileChunk(logicChunkIndex)
 		}
 		up.writableChunks[logicChunkIndex] = pageChunk
 	}
-	//if _, foundSealed := up.sealedChunks[logicChunkIndex]; foundSealed {
-	//	println("found already sealed chunk", logicChunkIndex)
-	//}
-	//if _, foundReading := up.activeReadChunks[logicChunkIndex]; foundReading {
-	//	println("found active read chunk", logicChunkIndex)
-	//}
-	n = pageChunk.WriteDataAt(p, off, tsNs)
+	n = pageChunk.WriteDataAt(p, off)
 	up.maybeMoveToSealed(pageChunk, logicChunkIndex)
 
 	return
 }
 
-func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
 	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
 
-	up.chunksLock.Lock()
-	defer func() {
-		up.readerCountCond.Signal()
-		up.chunksLock.Unlock()
-	}()
-
 	// read from sealed chunks first
+	up.sealedChunksLock.Lock()
 	sealedChunk, found := up.sealedChunks[logicChunkIndex]
 	if found {
-		maxStop = sealedChunk.chunk.ReadDataAt(p, off, tsNs)
+		sealedChunk.referenceCounter++
+	}
+	up.sealedChunksLock.Unlock()
+	if found {
+		maxStop = sealedChunk.chunk.ReadDataAt(p, off)
 		glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop)
+		sealedChunk.FreeReference(fmt.Sprintf("%s finish reading chunk %d", up.filepath, logicChunkIndex))
 	}
 
 	// read from writable chunks last
+	up.writableChunksLock.Lock()
+	defer up.writableChunksLock.Unlock()
 	writableChunk, found := up.writableChunks[logicChunkIndex]
 	if !found {
 		return
 	}
-	writableMaxStop := writableChunk.ReadDataAt(p, off, tsNs)
+	writableMaxStop := writableChunk.ReadDataAt(p, off)
 	glog.V(4).Infof("%s read writable memchunk [%d,%d)", up.filepath, off, writableMaxStop)
 	maxStop = max(maxStop, writableMaxStop)
 
@@ -147,8 +123,8 @@ func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64, tsNs int64) (maxS
 }
 
 func (up *UploadPipeline) FlushAll() {
-	up.chunksLock.Lock()
-	defer up.chunksLock.Unlock()
+	up.writableChunksLock.Lock()
+	defer up.writableChunksLock.Unlock()
 
 	for logicChunkIndex, memChunk := range up.writableChunks {
 		up.moveToSealed(memChunk, logicChunkIndex)
@@ -167,6 +143,8 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
 	atomic.AddInt32(&up.uploaderCount, 1)
 	glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, up.uploaderCount-1, up.uploaderCount)
 
+	up.sealedChunksLock.Lock()
+
 	if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found {
 		oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex))
 	}
@@ -177,8 +155,8 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
 	up.sealedChunks[logicChunkIndex] = sealedChunk
 	delete(up.writableChunks, logicChunkIndex)
 
-	// unlock before submitting the uploading jobs
-	up.chunksLock.Unlock()
+	up.sealedChunksLock.Unlock()
+
 	up.uploaders.Execute(func() {
 		// first add to the file chunks
 		sealedChunk.chunk.SaveContent(up.saveToStorageFn)
@@ -194,25 +172,24 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
 		up.uploaderCountCond.L.Unlock()
 
 		// wait for readers
-		up.chunksLock.Lock()
-		defer up.chunksLock.Unlock()
 		for up.IsLocked(logicChunkIndex) {
-			up.readerCountCond.Wait()
+			time.Sleep(59 * time.Millisecond)
 		}
 
 		// then remove from sealed chunks
+		up.sealedChunksLock.Lock()
+		defer up.sealedChunksLock.Unlock()
 		delete(up.sealedChunks, logicChunkIndex)
 		sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex))
 
 	})
-	up.chunksLock.Lock()
 }
 
 func (up *UploadPipeline) Shutdown() {
 	up.swapFile.FreeResource()
 
-	up.chunksLock.Lock()
-	defer up.chunksLock.Unlock()
+	up.sealedChunksLock.Lock()
+	defer up.sealedChunksLock.Unlock()
 	for logicChunkIndex, sealedChunk := range up.sealedChunks {
 		sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex))
 	}

weed/mount/page_writer/upload_pipeline_lock.go

@@ -10,8 +10,8 @@ func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) {
 	if stopOffset%up.ChunkSize > 0 {
 		stopLogicChunkIndex += 1
 	}
-	up.chunksLock.Lock()
-	defer up.chunksLock.Unlock()
+	up.activeReadChunksLock.Lock()
+	defer up.activeReadChunksLock.Unlock()
 	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
 		if count, found := up.activeReadChunks[i]; found {
 			up.activeReadChunks[i] = count + 1
@@ -27,8 +27,8 @@ func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
 	if stopOffset%up.ChunkSize > 0 {
 		stopLogicChunkIndex += 1
 	}
-	up.chunksLock.Lock()
-	defer up.chunksLock.Unlock()
+	up.activeReadChunksLock.Lock()
+	defer up.activeReadChunksLock.Unlock()
 	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
 		if count, found := up.activeReadChunks[i]; found {
 			if count == 1 {
@@ -41,6 +41,8 @@ func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
 }
 
 func (up *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool {
+	up.activeReadChunksLock.Lock()
+	defer up.activeReadChunksLock.Unlock()
 	if count, found := up.activeReadChunks[logicChunkIndex]; found {
 		return count > 0
 	}

weed/mount/page_writer/upload_pipeline_test.go

@@ -31,14 +31,14 @@ func writeRange(uploadPipeline *UploadPipeline, startOff, stopOff int64) {
 	p := make([]byte, 4)
 	for i := startOff / 4; i < stopOff/4; i += 4 {
 		util.Uint32toBytes(p, uint32(i))
-		uploadPipeline.SaveDataAt(p, i, false, 0)
+		uploadPipeline.SaveDataAt(p, i, false)
 	}
 }
 
 func confirmRange(t *testing.T, uploadPipeline *UploadPipeline, startOff, stopOff int64) {
 	p := make([]byte, 4)
 	for i := startOff; i < stopOff/4; i += 4 {
-		uploadPipeline.MaybeReadDataAt(p, i, 0)
+		uploadPipeline.MaybeReadDataAt(p, i)
 		x := util.BytesToUint32(p)
 		if x != uint32(i) {
 			t.Errorf("expecting %d found %d at offset [%d,%d)", i, x, i, i+4)

weed/mount/weedfs_attr.go

@@ -20,12 +20,12 @@ func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse
 	_, _, entry, status := wfs.maybeReadEntry(inode)
 	if status == fuse.OK {
 		out.AttrValid = 1
-		wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
+		wfs.setAttrByPbEntry(&out.Attr, inode, entry)
 		return status
 	} else {
 		if fh, found := wfs.fhmap.FindFileHandle(inode); found {
 			out.AttrValid = 1
-			wfs.setAttrByPbEntry(&out.Attr, inode, fh.entry.GetEntry(), true)
+			wfs.setAttrByPbEntry(&out.Attr, inode, fh.entry.GetEntry())
 			out.Nlink = 0
 			return fuse.OK
 		}
@@ -45,8 +45,8 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
 		return status
 	}
 	if fh != nil {
-		fh.entryLock.Lock()
-		defer fh.entryLock.Unlock()
+		//fh.entryLock.Lock()
+		//defer fh.entryLock.Unlock()
 	}
 
 	if size, ok := input.GetSize(); ok && entry != nil {
@@ -75,7 +75,7 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
 			// set the new chunks and reset entry cache
 			entry.Chunks = chunks
 			if fh != nil {
-				fh.entryChunkGroup.SetChunks(chunks)
+				fh.entryViewCache = nil
 			}
 		}
 		entry.Attributes.Mtime = time.Now().Unix()
@@ -114,11 +114,7 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
 	}
 
 	out.AttrValid = 1
-	size, includeSize := input.GetSize()
-	if includeSize {
-		out.Attr.Size = size
-	}
-	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry, !includeSize)
+	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)
 
 	if fh != nil {
 		fh.dirtyMetadata = true
@@ -143,22 +139,20 @@ func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
 	out.Nlink = 1
 }
 
-func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry, calculateSize bool) {
+func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry) {
 	out.Ino = inode
+	if entry.Attributes != nil && entry.Attributes.Inode != 0 {
+		out.Ino = entry.Attributes.Inode
+	}
+	out.Size = filer.FileSize(entry)
+	if entry.FileMode()&os.ModeSymlink != 0 {
+		out.Size = uint64(len(entry.Attributes.SymlinkTarget))
+	}
 	out.Blocks = (out.Size + blockSize - 1) / blockSize
 	setBlksize(out, blockSize)
 	if entry == nil {
 		return
 	}
-	if entry.Attributes != nil && entry.Attributes.Inode != 0 {
-		out.Ino = entry.Attributes.Inode
-	}
-	if calculateSize {
-		out.Size = filer.FileSize(entry)
-	}
-	if entry.FileMode()&os.ModeSymlink != 0 {
-		out.Size = uint64(len(entry.Attributes.SymlinkTarget))
-	}
 	out.Mtime = uint64(entry.Attributes.Mtime)
 	out.Ctime = uint64(entry.Attributes.Mtime)
 	out.Atime = uint64(entry.Attributes.Mtime)
@@ -200,7 +194,7 @@ func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.
 	out.Generation = 1
 	out.EntryValid = 1
 	out.AttrValid = 1
-	wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
+	wfs.setAttrByPbEntry(&out.Attr, inode, entry)
 }
 
 func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {

weed/mount/weedfs_dir_lookup.go

@@ -58,12 +58,12 @@ func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name strin
 	inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.Crtime.Unix(), localEntry.IsDirectory(), len(localEntry.HardLinkId) > 0, localEntry.Inode, true)
 
 	if fh, found := wfs.fhmap.FindFileHandle(inode); found {
-		fh.entryLock.Lock()
+		//fh.entryLock.Lock()
 		if entry := fh.GetEntry(); entry != nil {
 			glog.V(4).Infof("lookup opened file %s size %d", dirPath.Child(localEntry.Name()), filer.FileSize(entry))
 			localEntry = filer.FromPbEntry(string(dirPath), entry)
 		}
-		fh.entryLock.Unlock()
+		//fh.entryLock.Unlock()
 	}
 
 	wfs.outputFilerEntry(out, inode, localEntry)

weed/mount/weedfs_file_copy_range.go

@@ -1,8 +1,8 @@
 package mount
 
 import (
+	"context"
 	"net/http"
-	"time"
 
 	"github.com/hanwen/go-fuse/v2/fuse"
 
@@ -44,8 +44,8 @@ func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn)
 	}
 
 	// lock source and target file handles
-	fhOut.Lock()
-	defer fhOut.Unlock()
+	fhOut.orderedMutex.Acquire(context.Background(), 1)
+	defer fhOut.orderedMutex.Release(1)
 	fhOut.entryLock.Lock()
 	defer fhOut.entryLock.Unlock()
 
@@ -54,8 +54,8 @@ func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn)
 	}
 
 	if fhIn.fh != fhOut.fh {
-		fhIn.Lock()
-		defer fhIn.Unlock()
+		fhIn.orderedMutex.Acquire(context.Background(), 1)
+		defer fhIn.orderedMutex.Release(1)
 		fhIn.entryLock.Lock()
 		defer fhIn.entryLock.Unlock()
 	}
@@ -88,7 +88,7 @@ func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn)
 	// put data at the specified offset in target file
 	fhOut.dirtyPages.writerPattern.MonitorWriteAt(int64(in.OffOut), int(in.Len))
 	fhOut.entry.Content = nil
-	fhOut.dirtyPages.AddPage(int64(in.OffOut), data, fhOut.dirtyPages.writerPattern.IsSequentialMode(), time.Now().UnixNano())
+	fhOut.dirtyPages.AddPage(int64(in.OffOut), data, fhOut.dirtyPages.writerPattern.IsSequentialMode())
 	fhOut.entry.Attributes.FileSize = uint64(max(int64(in.OffOut)+totalRead, int64(fhOut.entry.Attributes.FileSize)))
 	fhOut.dirtyMetadata = true
 	written = uint32(totalRead)

weed/mount/weedfs_file_lseek.go

@@ -1,6 +1,7 @@
 package mount
 
 import (
+	"context"
 	"syscall"
 
 	"github.com/hanwen/go-fuse/v2/fuse"
@@ -35,10 +36,10 @@ func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekO
 	}
 
 	// lock the file until the proper offset was calculated
-	fh.Lock()
-	defer fh.Unlock()
-	fh.entryLock.Lock()
-	defer fh.entryLock.Unlock()
+	fh.orderedMutex.Acquire(context.Background(), 1)
+	defer fh.orderedMutex.Release(1)
+	//fh.entryLock.Lock()
+	//defer fh.entryLock.Unlock()
 
 	fileSize := int64(filer.FileSize(fh.GetEntry()))
 	offset := max(int64(in.Offset), 0)
@@ -55,8 +56,17 @@ func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekO
 		return ENXIO
 	}
 
+	// refresh view cache if necessary
+	if fh.entryViewCache == nil {
+		var err error
+		fh.entryViewCache, err = filer.NonOverlappingVisibleIntervals(fh.wfs.LookupFn(), fh.entry.GetChunks(), 0, fileSize)
+		if err != nil {
+			return fuse.EIO
+		}
+	}
+
 	// search chunks for the offset
-	found, offset := fh.entryChunkGroup.SearchChunks(offset, fileSize, in.Whence)
+	found, offset := searchChunks(fh, offset, fileSize, in.Whence)
 	if found {
 		out.Offset = uint64(offset)
 		return fuse.OK
@@ -72,3 +82,30 @@ func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekO
 
 	return fuse.OK
 }
+
+// searchChunks goes through all chunks to find the correct offset
+func searchChunks(fh *FileHandle, offset, fileSize int64, whence uint32) (found bool, out int64) {
+	chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, offset, fileSize)
+
+	for _, chunkView := range chunkViews {
+		if offset < chunkView.LogicOffset {
+			if whence == SEEK_HOLE {
+				out = offset
+			} else {
+				out = chunkView.LogicOffset
+			}
+
+			return true, out
+		}
+
+		if offset >= chunkView.LogicOffset && offset < chunkView.Offset+int64(chunkView.Size) && whence == SEEK_DATA {
+			out = offset
+
+			return true, out
+		}
+
+		offset += int64(chunkView.Size)
+	}
+
+	return
+}

weed/mount/weedfs_file_read.go

@@ -1,8 +1,7 @@
 package mount
 
 import (
-	"bytes"
-	"fmt"
+	"context"
 	"io"
 
 	"github.com/hanwen/go-fuse/v2/fuse"
@@ -41,8 +40,8 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse
 		return nil, fuse.ENOENT
 	}
 
-	fh.Lock()
-	defer fh.Unlock()
+	fh.orderedMutex.Acquire(context.Background(), 1)
+	defer fh.orderedMutex.Release(1)
 
 	offset := int64(in.Offset)
 	totalRead, err := readDataByFileHandle(buff, fh, offset)
@@ -51,23 +50,6 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse
 		return nil, fuse.EIO
 	}
 
-	if IsDebugFileReadWrite {
-		// print(".")
-		mirrorData := make([]byte, totalRead)
-		fh.mirrorFile.ReadAt(mirrorData, offset)
-		if bytes.Compare(mirrorData, buff[:totalRead]) != 0 {
-
-			againBuff := make([]byte, len(buff))
-			againRead, _ := readDataByFileHandle(buff, fh, offset)
-			againCorrect := bytes.Compare(mirrorData, againBuff[:againRead]) == 0
-			againSame := bytes.Compare(buff[:totalRead], againBuff[:againRead]) == 0
-
-			fmt.Printf("\ncompare %v [%d,%d) size:%d againSame:%v againCorrect:%v\n", fh.mirrorFile.Name(), offset, offset+totalRead, totalRead, againSame, againCorrect)
-			//fmt.Printf("read mirrow data: %v\n", mirrorData)
-			//fmt.Printf("read actual data: %v\n", buff[:totalRead])
-		}
-	}
-
 	return fuse.ReadResultData(buff[:totalRead]), fuse.OK
 }
 
@@ -77,9 +59,9 @@ func readDataByFileHandle(buff []byte, fhIn *FileHandle, offset int64) (int64, e
 	fhIn.lockForRead(offset, size)
 	defer fhIn.unlockForRead(offset, size)
 
-	n, tsNs, err := fhIn.readFromChunks(buff, offset)
+	n, err := fhIn.readFromChunks(buff, offset)
 	if err == nil || err == io.EOF {
-		maxStop := fhIn.readFromDirtyPages(buff, offset, tsNs)
+		maxStop := fhIn.readFromDirtyPages(buff, offset)
 		n = max(maxStop-offset, n)
 	}
 	if err == io.EOF {

weed/mount/weedfs_file_sync.go

@@ -55,6 +55,9 @@ func (wfs *WFS) Flush(cancel <-chan struct{}, in *fuse.FlushIn) fuse.Status {
 		return fuse.ENOENT
 	}
 
+	fh.orderedMutex.Acquire(context.Background(), 1)
+	defer fh.orderedMutex.Release(1)
+
 	return wfs.doFlush(fh, in.Uid, in.Gid)
 }
 
@@ -84,14 +87,14 @@ func (wfs *WFS) Fsync(cancel <-chan struct{}, in *fuse.FsyncIn) (code fuse.Statu
 		return fuse.ENOENT
 	}
 
+	fh.orderedMutex.Acquire(context.Background(), 1)
+	defer fh.orderedMutex.Release(1)
+
 	return wfs.doFlush(fh, in.Uid, in.Gid)
 
 }
 
 func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
-	fh.Lock()
-	defer fh.Unlock()
-
 	// flush works at fh level
 	fileFullPath := fh.FullPath()
 	dir, name := fileFullPath.DirAndName()
@@ -114,8 +117,9 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
 	}
 
 	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		fh.entryLock.Lock()
-		defer fh.entryLock.Unlock()
+
+		//fh.entryLock.Lock()
+		//defer fh.entryLock.Unlock()
 
 		entry := fh.GetEntry()
 		if entry == nil {
@@ -145,9 +149,9 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
 		}
 
 		glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.GetChunks()))
-		//for i, chunk := range entry.GetChunks() {
-		//	glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fileFullPath, i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
-		//}
+		for i, chunk := range entry.GetChunks() {
+			glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fileFullPath, i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
+		}
 
 		manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.GetChunks())
 
@@ -181,9 +185,5 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
 		return fuse.EIO
 	}
 
-	if IsDebugFileReadWrite {
-		fh.mirrorFile.Sync()
-	}
-
 	return fuse.OK
 }

weed/mount/weedfs_file_write.go

@@ -1,10 +1,10 @@
 package mount
 
 import (
+	"context"
 	"github.com/hanwen/go-fuse/v2/fuse"
 	"net/http"
 	"syscall"
-	"time"
 )
 
 /**
@@ -46,10 +46,8 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
 
 	fh.dirtyPages.writerPattern.MonitorWriteAt(int64(in.Offset), int(in.Size))
 
-	tsNs := time.Now().UnixNano()
-
-	fh.Lock()
-	defer fh.Unlock()
+	fh.orderedMutex.Acquire(context.Background(), 1)
+	defer fh.orderedMutex.Release(1)
 
 	entry := fh.GetEntry()
 	if entry == nil {
@@ -61,7 +59,7 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
 	entry.Attributes.FileSize = uint64(max(offset+int64(len(data)), int64(entry.Attributes.FileSize)))
 	// glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data))
 
-	fh.dirtyPages.AddPage(offset, data, fh.dirtyPages.writerPattern.IsSequentialMode(), tsNs)
+	fh.dirtyPages.AddPage(offset, data, fh.dirtyPages.writerPattern.IsSequentialMode())
 
 	written = uint32(len(data))
 
@@ -72,10 +70,5 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
 
 	fh.dirtyMetadata = true
 
-	if IsDebugFileReadWrite {
-		// print("+")
-		fh.mirrorFile.WriteAt(data, offset)
-	}
-
 	return written, fuse.OK
 }

weed/mount/weedfs_write.go

@@ -13,7 +13,7 @@ import (
 
 func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFunctionType {
 
-	return func(reader io.Reader, filename string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error) {
+	return func(reader io.Reader, filename string, offset int64) (chunk *filer_pb.FileChunk, err error) {
 
 		fileId, uploadResult, err, data := operation.UploadWithRetry(
 			wfs,
@@ -56,7 +56,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFun
 			wfs.chunkCache.SetChunk(fileId, data)
 		}
 
-		chunk = uploadResult.ToPbFileChunk(fileId, offset, tsNs)
+		chunk = uploadResult.ToPbFileChunk(fileId, offset)
 		return chunk, nil
 	}
 }

weed/mount/weedfs_xattr.go

@@ -111,8 +111,8 @@ func (wfs *WFS) SetXAttr(cancel <-chan struct{}, input *fuse.SetXAttrIn, attr st
 		return fuse.ENOENT
 	}
 	if fh != nil {
-		fh.entryLock.Lock()
-		defer fh.entryLock.Unlock()
+		//fh.entryLock.Lock()
+		//defer fh.entryLock.Unlock()
 	}
 
 	if entry.Extended == nil {
@@ -189,8 +189,8 @@ func (wfs *WFS) RemoveXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr
 		return fuse.OK
 	}
 	if fh != nil {
-		fh.entryLock.Lock()
-		defer fh.entryLock.Unlock()
+		//fh.entryLock.Lock()
+		//defer fh.entryLock.Unlock()
 	}
 
 	if entry.Extended == nil {

weed/mount/wfs_filer_client.go

@@ -22,7 +22,7 @@ func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFile
 		for x := 0; x < n; x++ {
 
 			filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress()
-			err = pb.WithGrpcClient(streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
+			err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
 				client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 				return fn(client)
 			}, filerGrpcAddress, false, wfs.option.GrpcDialOption)