Compare commits

..
17 Commits
Author SHA1 Message Date
chrislu 2f86f10fe1 Merge branch 'master' into message_send 2022-09-25 23:38:27 -07:00
chrislu 264c138c1d minor 2022-09-25 19:00:34 -07:00
chrislu 24317a9763 async wait 2022-09-25 17:57:26 -07:00
chrislu 3da8d7309a Merge branch 'master' into message_send 2022-09-25 17:46:12 -07:00
chrislu 648b33beb0 add future list 2022-09-25 17:42:23 -07:00
chrislu 1484cb224f Merge branch 'master' into message_send 2022-09-25 13:50:02 -07:00
chrislu 600d2f92a4 add message pipeline 2022-09-25 11:19:05 -07:00
chrislu 15b6c7a6f0 remove file 2022-09-22 23:21:47 -07:00
chrislu bfd0601fd3 refactoring 2022-09-22 23:21:13 -07:00
chrislu 977c8e7830 add todo 2022-09-18 16:49:10 -07:00
chrislu e264fa6e28 volume server: rename readBufferSize to readBufferSizeMB 2022-09-17 10:50:06 -07:00
chrislu dcbded20bb minor 2022-09-17 10:24:47 -07:00
chrislu 1c79301f89 Merge branch 'master' into message_send 2022-09-17 10:18:14 -07:00
chrislu 956ce6416f Merge branch 'master' into message_send 2022-09-14 00:07:27 -07:00
chrislu 3f4400dc03 Merge branch 'master' into message_send 2022-09-05 10:47:13 -07:00
chrislu daa086fd72 segment infos 2022-09-04 18:14:46 -07:00
chrislu 7e33915a26 qsend to broker 2022-09-04 01:29:01 -07:00
326 changed files with 4697 additions and 118011 deletions
+6 -6
View File
@@ -38,13 +38,13 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -87,13 +87,13 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+3 -3
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+3 -3
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+3 -3
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+3 -3
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+3 -3
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
+7 -7
View File
@@ -16,11 +16,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
@@ -33,30 +33,30 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
with:
buildkitd-flags: "--debug"
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
-
name: Build
uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+7 -7
View File
@@ -17,11 +17,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
@@ -34,30 +34,30 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
with:
buildkitd-flags: "--debug"
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
-
name: Build
uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+6 -6
View File
@@ -16,11 +16,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
@@ -34,20 +34,20 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+6 -6
View File
@@ -17,11 +17,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
@@ -35,20 +35,20 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+6 -6
View File
@@ -17,11 +17,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
@@ -35,20 +35,20 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+6 -6
View File
@@ -16,11 +16,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
@@ -34,20 +34,20 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+6 -6
View File
@@ -16,11 +16,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea # v3
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
@@ -34,20 +34,20 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v1
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # v2
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+2 -2
View File
@@ -9,6 +9,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
- name: 'Dependency Review'
uses: actions/dependency-review-action@c090f4e553673e6e505ea70d6a95362ee12adb94
uses: actions/dependency-review-action@23d1ffffb6fa5401173051ec21eba8c35242733f
+2 -2
View File
@@ -24,13 +24,13 @@ jobs:
timeout-minutes: 15
steps:
- name: Set up Go 1.x
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
with:
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Install dependencies
run: |
+2 -2
View File
@@ -21,13 +21,13 @@ jobs:
steps:
- name: Set up Go 1.x
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
with:
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Get dependencies
run: |
-74
View File
@@ -1,74 +0,0 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
- Using welcoming and inclusive language
- Being respectful of differing viewpoints and experiences
- Gracefully accepting constructive criticism
- Focusing on what is best for the community
- Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
- The use of sexualized language or imagery and unwelcome sexual attention or
advances
- Trolling, insulting/derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or electronic
address, without explicit permission
- Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at <enteremailhere>. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at [http://contributor-covenant.org/version/1/4][version]
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
+17 -17
View File
@@ -39,63 +39,63 @@ s3tests_build:
docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
dev: build
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_race: binary_race
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_tls: build certstrap
ENV_FILE="tls.env" docker compose -f compose/local-dev-compose.yml -p seaweedfs up
ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_mount: build
docker compose -f compose/local-mount-compose.yml -p seaweedfs up
docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
run_image: build
docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local
profile_mount: build
docker compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
k8s: build
docker compose -f compose/local-k8s-compose.yml -p seaweedfs up
docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
dev_registry: build
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
docker-compose -f compose/local-registry-compose.yml -p seaweedfs up
dev_replicate: build
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
dev_auditlog: build
docker compose -f compose/local-auditlog-compose.yml -p seaweedfs up
docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
dev_nextcloud: build
docker compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
docker-compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
cluster: build
docker compose -f compose/local-cluster-compose.yml -p seaweedfs up
docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
2clusters: build
docker compose -f compose/local-clusters-compose.yml -p seaweedfs up
docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
2mount: build
docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
docker-compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
hashicorp_raft: build
docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
s3tests: build s3tests_build
docker compose -f compose/local-s3tests-compose.yml -p seaweedfs up
docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
filer_etcd: build
docker stack deploy -c compose/swarm-etcd.yml fs
test_etcd: build
docker compose -f compose/test-etcd-filer.yml -p seaweedfs up
docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
test_ydb: tags = ydb
test_ydb: build
export
docker compose -f compose/test-ydb-filer.yml -p seaweedfs up
docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up
clean:
rm ./weed
+2 -12
View File
@@ -1,15 +1,5 @@
# Docker
## Compose V2
SeaweedFS now uses the `v2` syntax `docker compose`
If you rely on using Docker Compose as docker-compose (with a hyphen), you can set up Compose V2 to act as a drop-in replacement of the previous docker-compose. Refer to the [Installing Compose](https://docs.docker.com/compose/install/) section for detailed instructions on upgrading.
Confirm your system has docker compose v2 with a version check
```bash
$ docker compose version
Docker Compose version v2.10.2
```
## Try it out
@@ -17,7 +7,7 @@ Docker Compose version v2.10.2
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml
docker compose -f seaweedfs-compose.yml -p seaweedfs up
docker-compose -f seaweedfs-compose.yml -p seaweedfs up
```
@@ -27,7 +17,7 @@ docker compose -f seaweedfs-compose.yml -p seaweedfs up
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml
docker compose -f seaweedfs-dev-compose.yml -p seaweedfs up
docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
```
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
s3:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master0:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
server1:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master0:
@@ -6,7 +6,7 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
volumes:
- ./master/0:/data
environment:
@@ -18,7 +18,7 @@ services:
ports:
- 9334:9334
- 19334:19334
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
volumes:
- ./master/1:/data
environment:
@@ -30,7 +30,7 @@ services:
ports:
- 9335:9335
- 19335:19335
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
volumes:
- ./master/2:/data
environment:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
etcd:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
ydb:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '3.9'
version: '2'
services:
master:
+7 -7
View File
@@ -3,10 +3,10 @@ CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret';
GRANT ALL PRIVILEGES ON seaweedfs.* TO 'seaweedfs'@'%';
FLUSH PRIVILEGES;
USE seaweedfs;
CREATE TABLE IF NOT EXISTS `filemeta` (
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
`meta` LONGBLOB,
PRIMARY KEY (`dirhash`, `name`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
CREATE TABLE IF NOT EXISTS filemeta (
dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
name VARCHAR(1000) COMMENT 'directory or file name',
directory TEXT COMMENT 'full path to parent directory',
meta LONGBLOB,
PRIMARY KEY (dirhash, name)
) DEFAULT CHARSET=utf8;
+86 -93
View File
@@ -3,18 +3,18 @@ module github.com/seaweedfs/seaweedfs
go 1.19
require (
cloud.google.com/go v0.105.0 // indirect
cloud.google.com/go/pubsub v1.28.0
cloud.google.com/go/storage v1.28.1
cloud.google.com/go v0.104.0 // indirect
cloud.google.com/go/pubsub v1.25.1
cloud.google.com/go/storage v1.26.0
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/Shopify/sarama v1.37.2
github.com/aws/aws-sdk-go v1.44.175
github.com/Shopify/sarama v1.36.0
github.com/aws/aws-sdk-go v1.44.100
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/disintegration/imaging v1.6.2
@@ -27,12 +27,12 @@ require (
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fclairamb/ftpserverlib v0.20.0
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fclairamb/ftpserverlib v0.19.1
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redsync/redsync/v4 v4.7.1
github.com/go-sql-driver/mysql v1.7.0
github.com/go-redsync/redsync/v4 v4.6.0
github.com/go-sql-driver/mysql v1.6.0
github.com/go-zookeeper/zk v1.0.2 // indirect
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
github.com/golang-jwt/jwt v3.2.2+incompatible
@@ -40,13 +40,13 @@ require (
github.com/golang/protobuf v1.5.2
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/uuid v1.3.0
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
@@ -57,16 +57,16 @@ require (
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/reedsolomon v1.11.3
github.com/klauspost/compress v1.15.9 // indirect
github.com/klauspost/reedsolomon v1.11.0
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.7
github.com/linxGnu/grocksdb v1.7.10
github.com/linxGnu/grocksdb v1.7.7
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -77,23 +77,23 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/posener/complete v1.2.3
github.com/pquerna/cachecontrol v0.1.0
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/client_golang v1.13.0
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.9.0
github.com/prometheus/procfs v0.8.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/seaweedfs/goexif v2.0.0+incompatible
github.com/seaweedfs/raft v1.1.0
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/afero v1.9.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.14.0
github.com/stretchr/testify v1.8.1
github.com/spf13/viper v1.13.0
github.com/stretchr/testify v1.8.0
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
github.com/tidwall/gjson v1.14.4
github.com/tidwall/gjson v1.14.3
github.com/tidwall/match v1.1.1
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@@ -106,85 +106,84 @@ require (
github.com/xdg-go/scram v1.1.1 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.etcd.io/etcd/client/v3 v3.5.6
go.mongodb.org/mongo-driver v1.11.1
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.27.0
gocloud.dev/pubsub/natspubsub v0.27.0
gocloud.dev/pubsub/rabbitpubsub v0.27.0
golang.org/x/crypto v0.1.0 // indirect
go.etcd.io/etcd/client/v3 v3.5.5
go.mongodb.org/mongo-driver v1.10.2
go.opencensus.io v0.23.0 // indirect
gocloud.dev v0.26.0
gocloud.dev/pubsub/natspubsub v0.26.0
gocloud.dev/pubsub/rabbitpubsub v0.26.0
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/net v0.4.0
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
golang.org/x/sys v0.4.0
golang.org/x/text v0.5.0 // indirect
golang.org/x/tools v0.4.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.106.0
golang.org/x/net v0.0.0-20220909164309-bea034e7d591
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
golang.org/x/sys v0.0.0-20220818161305-2296e01440c6
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.12
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/api v0.96.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
google.golang.org/grpc v1.51.0
google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
google.golang.org/grpc v1.49.0
google.golang.org/protobuf v1.28.1
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.22.2 // indirect
modernc.org/cc/v3 v3.38.1 // indirect
modernc.org/ccgo/v3 v3.16.9 // indirect
modernc.org/libc v1.19.0 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.4.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/sqlite v1.20.1
modernc.org/sqlite v1.19.1
modernc.org/strutil v1.1.3
modernc.org/token v1.0.1 // indirect
)
require (
github.com/Jille/raft-grpc-transport v1.3.0
github.com/arangodb/go-driver v1.4.1
github.com/armon/go-metrics v0.4.1
github.com/Jille/raft-grpc-transport v1.2.0
github.com/arangodb/go-driver v1.3.3
github.com/fluent/fluent-logger-golang v1.9.0
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
github.com/google/flatbuffers v2.0.8+incompatible
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
github.com/hashicorp/raft v1.3.11
github.com/hashicorp/raft v1.3.10
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/rabbitmq/amqp091-go v1.5.0
github.com/schollz/progressbar/v3 v3.12.2
github.com/tikv/client-go/v2 v2.0.4
github.com/tikv/client-go/v2 v2.0.1
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.1
github.com/ydb-platform/ydb-go-sdk/v3 v3.38.0
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
)
require (
cloud.google.com/go/compute v1.14.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.8.0 // indirect
cloud.google.com/go/compute v1.7.0 // indirect
cloud.google.com/go/iam v0.3.0 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.9 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.17.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.13 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.10 // indirect
github.com/aws/smithy-go v1.12.0 // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fclairamb/go-log v0.4.1 // indirect
github.com/fclairamb/go-log v0.3.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
@@ -195,49 +194,43 @@ require (
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/montanaflynn/stats v0.6.6 // indirect
github.com/nats-io/nats.go v1.16.0 // indirect
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/philhofer/fwd v1.1.1 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect
github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stathat/consistent v1.0.0 // indirect
github.com/subosito/gotenv v1.4.1 // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 // indirect
github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 // indirect
github.com/tinylib/msgp v1.1.6 // indirect
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220916101045-1c4d1accabb6 // indirect
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/api/v3 v3.5.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.6 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.22.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.3.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
lukechampine.com/uint128 v1.1.1 // indirect
)
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
+252 -1307
View File
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "latest"
version: "latest"
appVersion: "3.29"
version: "3.29"
+7 -7
View File
@@ -14,13 +14,13 @@ with ENV.
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
This database should be pre-configured and initialized by running:
```sql
CREATE TABLE IF NOT EXISTS `filemeta` (
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
`meta` LONGBLOB,
PRIMARY KEY (`dirhash`, `name`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
CREATE TABLE IF NOT EXISTS filemeta (
dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
name VARCHAR(1000) BINARY COMMENT 'directory or file name',
directory TEXT BINARY COMMENT 'full path to parent directory',
meta LONGBLOB,
PRIMARY KEY (dirhash, name)
) DEFAULT CHARSET=utf8;
```
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
@@ -101,12 +101,7 @@ spec:
- "/bin/sh"
- "-ec"
- |
exec /usr/bin/weed \
{{- if eq .Values.filer.logs.type "hostPath" }}
-logdir=/logs \
{{- else }}
-logtostderr=true \
{{- end }}
exec /usr/bin/weed -logdir=/logs \
{{- if .Values.filer.loggingOverrideLevel }}
-v={{ .Values.filer.loggingOverrideLevel }} \
{{- else }}
@@ -230,13 +225,13 @@ spec:
{{- if eq .Values.filer.logs.type "hostPath" }}
- name: seaweedfs-filer-log-volume
hostPath:
path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer
path: /storage/logs/seaweedfs/filer
type: DirectoryOrCreate
{{- end }}
{{- if eq .Values.filer.data.type "hostPath" }}
- name: data-filer
hostPath:
path: {{ .Values.filer.data.hostPathPrefix }}/filer_store
path: /storage/filer_store
type: DirectoryOrCreate
{{- end }}
- name: db-schema-config-volume
@@ -90,12 +90,7 @@ spec:
- "/bin/sh"
- "-ec"
- |
exec /usr/bin/weed \
{{- if eq .Values.master.logs.type "hostPath" }}
-logdir=/logs \
{{- else }}
-logtostderr=true \
{{- end }}
exec /usr/bin/weed -logdir=/logs \
{{- if .Values.master.loggingOverrideLevel }}
-v={{ .Values.master.loggingOverrideLevel }} \
{{- else }}
@@ -201,13 +196,13 @@ spec:
{{- if eq .Values.master.logs.type "hostPath" }}
- name: seaweedfs-master-log-volume
hostPath:
path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master
path: /storage/logs/seaweedfs/master
type: DirectoryOrCreate
{{- end }}
{{- if eq .Values.master.data.type "hostPath" }}
- name: data-{{ .Release.Namespace }}
hostPath:
path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/
path: /ssd/seaweed-master/
type: DirectoryOrCreate
{{- end }}
{{- if .Values.global.enableSecurity }}
@@ -72,19 +72,13 @@ spec:
- "/bin/sh"
- "-ec"
- |
exec /usr/bin/weed \
{{- if eq .Values.s3.logs.type "hostPath" }}
-logdir=/logs \
{{- else }}
-logtostderr=true \
{{- end }}
exec /usr/bin/weed -logdir=/logs \
{{- if .Values.s3.loggingOverrideLevel }}
-v={{ .Values.s3.loggingOverrideLevel }} \
{{- else }}
-v={{ .Values.global.loggingLevel }} \
{{- end }}
s3 \
-ip.bind={{ .Values.s3.bindAddress }} \
-port={{ .Values.s3.port }} \
{{- if .Values.s3.metricsPort }}
-metricsPort {{ .Values.s3.metricsPort }} \
@@ -173,7 +167,7 @@ spec:
{{- if eq .Values.s3.logs.type "hostPath" }}
- name: logs
hostPath:
path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3
path: /storage/logs/seaweedfs/s3
type: DirectoryOrCreate
{{- end }}
{{- if .Values.global.enableSecurity }}
@@ -34,7 +34,7 @@ spec:
restartPolicy: {{ default .Values.global.restartPolicy .Values.volume.restartPolicy }}
{{- if .Values.volume.tolerations }}
tolerations:
{{ tpl .Values.volume.tolerations . | nindent 8 | trim }}
{{ tpl .Values.volume.tolerations . | nindent 8 | trim }}
{{- end }}
{{- if .Values.global.imagePullSecrets }}
imagePullSecrets:
@@ -93,12 +93,7 @@ spec:
- "/bin/sh"
- "-ec"
- |
exec /usr/bin/weed \
{{- if eq .Values.volume.logs.type "hostPath" }}
-logdir=/logs \
{{- else }}
-logtostderr=true \
{{- end }}
exec /usr/bin/weed -logdir=/logs \
{{- if .Values.volume.loggingOverrideLevel }}
-v={{ .Values.volume.loggingOverrideLevel }} \
{{- else }}
@@ -211,19 +206,19 @@ spec:
{{- if eq .Values.volume.data.type "hostPath" }}
- name: data
hostPath:
path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
path: /storage/object_store/
type: DirectoryOrCreate
{{- end }}
{{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
- name: idx
hostPath:
path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
path: /ssd/seaweedfs-volume-idx/
type: DirectoryOrCreate
{{- end }}
{{- if eq .Values.volume.logs.type "hostPath" }}
- name: logs
hostPath:
path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
path: /storage/logs/seaweedfs/volume
type: DirectoryOrCreate
{{- end }}
{{- if .Values.global.enableSecurity }}
+1 -10
View File
@@ -68,13 +68,11 @@ master:
type: "hostPath"
size: ""
storageClass: ""
hostPathPrefix: /ssd
logs:
type: "hostPath"
size: ""
storageClass: ""
hostPathPrefix: /storage
initContainers: ""
@@ -178,19 +176,15 @@ volume:
type: "hostPath"
size: ""
storageClass: ""
hostPathPrefix: /storage
idx:
type: "hostPath"
size: ""
storageClass: ""
hostPathPrefix: /ssd
logs:
type: "hostPath"
size: ""
storageClass: ""
hostPathPrefix: /storage
# limit background compaction or copying speed in mega bytes per second
compactionMBps: "50"
@@ -314,13 +308,11 @@ filer:
type: "hostPath"
size: ""
storageClass: ""
hostPathPrefix: /storage
logs:
type: "hostPath"
size: ""
storageClass: ""
hostPathPrefix: /storage
initContainers: ""
@@ -426,7 +418,6 @@ s3:
imageTag: null
restartPolicy: null
replicas: 1
bindAddress: 0.0.0.0
port: 8333
metricsPort: 9327
loggingOverrideLevel: null
@@ -472,7 +463,7 @@ s3:
type: "hostPath"
size: ""
storageClass: ""
hostPathPrefix: /storage
certificates:
commonName: "SeaweedFS CA"
+3 -3
View File
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>3.33</version>
<version>3.13</version>
<parent>
<groupId>org.sonatype.oss</groupId>
@@ -14,7 +14,7 @@
</parent>
<properties>
<protobuf.version>3.16.3</protobuf.version>
<protobuf.version>3.16.1</protobuf.version>
<!-- follow https://github.com/grpc/grpc-java -->
<grpc.version>1.23.0</grpc.version>
<guava.version>30.0-jre</guava.version>
@@ -159,7 +159,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<version>2.9.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
+2 -2
View File
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>3.33</version>
<version>3.13</version>
<parent>
<groupId>org.sonatype.oss</groupId>
@@ -154,7 +154,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<version>2.9.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
+2 -2
View File
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>3.33</version>
<version>3.13</version>
<parent>
<groupId>org.sonatype.oss</groupId>
@@ -128,7 +128,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<version>2.9.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
@@ -83,7 +83,7 @@ public class ReadChunks {
prevX,
point.x,
chunk.getFileId(),
chunk.getModifiedTsNs(),
chunk.getMtime(),
prevX - chunk.getOffset(),
chunk.getOffset() == prevX && chunk.getSize() == prevX - startPoint.x,
chunk.getCipherKey().toByteArray(),
@@ -100,7 +100,7 @@ public class ReadChunks {
public Point(long x, FilerProto.FileChunk chunk, boolean isStart) {
this.x = x;
this.ts = chunk.getModifiedTsNs();
this.ts = chunk.getMtime();
this.chunk = chunk;
this.isStart = isStart;
}
@@ -119,8 +119,9 @@ public class SeaweedInputStream extends InputStream {
long bytesRead = 0;
int len = buf.remaining();
if (this.position< Integer.MAX_VALUE && (this.position + len )<= entry.getContent().size()) {
entry.getContent().substring((int)this.position, (int)(this.position + len)).copyTo(buf);
int start = (int) this.position;
if (start + len <= entry.getContent().size()) {
entry.getContent().substring(start, start + len).copyTo(buf);
} else {
bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
}
@@ -14,9 +14,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.SecureRandom;
import java.security.MessageDigest;
import java.util.List;
import java.util.Base64;
public class SeaweedWrite {
@@ -98,7 +96,7 @@ public class SeaweedWrite {
.setFileId(fileId)
.setOffset(offset)
.setSize(bytesLength)
.setModifiedTsNs(System.nanoTime())
.setMtime(System.currentTimeMillis() / 10000L)
.setETag(etag)
.setCipherKey(cipherKeyString);
}
@@ -125,20 +123,13 @@ public class SeaweedWrite {
final byte[] bytes,
final long bytesOffset, final long bytesLength,
byte[] cipherKey) throws IOException {
MessageDigest md = null;
try {
md = MessageDigest.getInstance("MD5");
} catch (java.security.NoSuchAlgorithmException e) {
}
InputStream inputStream = null;
if (cipherKey == null || cipherKey.length == 0) {
md.update(bytes, (int) bytesOffset, (int) bytesLength);
inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
} else {
try {
byte[] encryptedBytes = SeaweedCipher.encrypt(bytes, (int) bytesOffset, (int) bytesLength, cipherKey);
md.update(encryptedBytes);
inputStream = new ByteArrayInputStream(encryptedBytes, 0, encryptedBytes.length);
} catch (Exception e) {
throw new IOException("fail to encrypt data", e);
@@ -149,7 +140,6 @@ public class SeaweedWrite {
if (auth != null && auth.length() != 0) {
post.addHeader("Authorization", "BEARER " + auth);
}
post.addHeader("Content-MD5", Base64.getEncoder().encodeToString(md.digest()));
post.setEntity(MultipartEntityBuilder.create()
.setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
@@ -159,13 +149,6 @@ public class SeaweedWrite {
CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
try {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
if (response.getEntity().getContentType() != null && response.getEntity().getContentType().getValue().equals("application/json")) {
throw new IOException(EntityUtils.toString(response.getEntity(), "UTF-8"));
} else {
throw new IOException(response.getStatusLine().getReasonPhrase());
}
}
String etag = response.getLastHeader("ETag").getValue();
+1 -1
View File
@@ -132,7 +132,7 @@ message FileChunk {
string file_id = 1; // to be deprecated
int64 offset = 2;
uint64 size = 3;
int64 modified_ts_ns = 4;
int64 mtime = 4;
string e_tag = 5;
string source_file_id = 6; // to be deprecated
FileId fid = 7;
@@ -17,13 +17,13 @@ public class SeaweedReadTest {
.setFileId("aaa")
.setOffset(0)
.setSize(100)
.setModifiedTsNs(1000)
.setMtime(1000)
.build());
chunks.add(FilerProto.FileChunk.newBuilder()
.setFileId("bbb")
.setOffset(100)
.setSize(133)
.setModifiedTsNs(2000)
.setMtime(2000)
.build());
List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(null, chunks);
@@ -70,31 +70,31 @@ public class SeaweedReadTest {
.setFileId("a")
.setOffset(0)
.setSize(100)
.setModifiedTsNs(1)
.setMtime(1)
.build());
chunks.add(FilerProto.FileChunk.newBuilder()
.setFileId("b")
.setOffset(50)
.setSize(100)
.setModifiedTsNs(2)
.setMtime(2)
.build());
chunks.add(FilerProto.FileChunk.newBuilder()
.setFileId("c")
.setOffset(200)
.setSize(50)
.setModifiedTsNs(3)
.setMtime(3)
.build());
chunks.add(FilerProto.FileChunk.newBuilder()
.setFileId("d")
.setOffset(250)
.setSize(50)
.setModifiedTsNs(4)
.setMtime(4)
.build());
chunks.add(FilerProto.FileChunk.newBuilder()
.setFileId("e")
.setOffset(175)
.setSize(100)
.setModifiedTsNs(5)
.setMtime(5)
.build());
List<SeaweedRead.VisibleInterval> visibleIntervals = ReadChunks.readResolvedChunks(chunks);
@@ -161,7 +161,7 @@ public class SeaweedReadTest {
.setFileId("")
.setOffset(start)
.setSize(size)
.setModifiedTsNs(ts)
.setMtime(ts)
.build();
}
}
+2 -2
View File
@@ -11,13 +11,13 @@
<dependency>
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>3.33</version>
<version>3.13</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-hadoop2-client</artifactId>
<version>3.33</version>
<version>3.13</version>
<scope>compile</scope>
</dependency>
<dependency>
+1 -1
View File
@@ -31,7 +31,7 @@
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.10.0</version>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+43 -51
View File
@@ -108,7 +108,7 @@
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<version>2.9.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
@@ -124,7 +124,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.2.4</version>
<version>2.10.1</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -132,11 +132,11 @@
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-yarn-api</artifactId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-yarn-client</artifactId>
<artifactId>hadoop-yarn-api</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
@@ -156,7 +156,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.4</version>
<version>2.10.1</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -167,6 +167,10 @@
<artifactId>commons-math3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>xmlenc</artifactId>
<groupId>xmlenc</groupId>
</exclusion>
<exclusion>
<artifactId>commons-io</artifactId>
<groupId>commons-io</groupId>
@@ -180,28 +184,20 @@
<groupId>commons-collections</groupId>
</exclusion>
<exclusion>
<artifactId>javax.servlet-api</artifactId>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>javax.activation-api</artifactId>
<groupId>javax.activation</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-server</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty</artifactId>
<groupId>org.mortbay.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-util</artifactId>
<groupId>org.eclipse.jetty</groupId>
<groupId>org.mortbay.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-servlet</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-webapp</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<groupId>org.mortbay.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jsp-api</artifactId>
@@ -211,10 +207,6 @@
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-servlet</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-json</artifactId>
<groupId>com.sun.jersey</groupId>
@@ -224,37 +216,49 @@
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>jets3t</artifactId>
<groupId>net.java.dev.jets3t</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang</artifactId>
<groupId>commons-lang</groupId>
</exclusion>
<exclusion>
<artifactId>commons-configuration</artifactId>
<groupId>commons-configuration</groupId>
</exclusion>
<exclusion>
<artifactId>commons-digester</artifactId>
<groupId>commons-digester</groupId>
</exclusion>
<exclusion>
<artifactId>commons-beanutils</artifactId>
<groupId>commons-beanutils</groupId>
</exclusion>
<exclusion>
<artifactId>commons-configuration2</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-text</artifactId>
<groupId>org.apache.commons</groupId>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-reload4j</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>jackson-core-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-mapper-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>re2j</artifactId>
<groupId>com.google.re2j</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-auth</artifactId>
<groupId>org.apache.hadoop</groupId>
@@ -283,14 +287,6 @@
<artifactId>commons-compress</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>kerb-simplekdc</artifactId>
<groupId>org.apache.kerby</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>stax2-api</artifactId>
<groupId>org.codehaus.woodstox</groupId>
@@ -299,10 +295,6 @@
<artifactId>woodstox-core</artifactId>
<groupId>com.fasterxml.woodstox</groupId>
</exclusion>
<exclusion>
<artifactId>dnsjava</artifactId>
<groupId>dnsjava</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-annotations</artifactId>
<groupId>org.apache.hadoop</groupId>
@@ -317,7 +309,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
<seaweedfs.client.version>3.33</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
<seaweedfs.client.version>3.13</seaweedfs.client.version>
<hadoop.version>2.10.1</hadoop.version>
</properties>
</project>
+2 -2
View File
@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>3.33</seaweedfs.client.version>
<seaweedfs.client.version>3.13</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
@@ -129,7 +129,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<version>2.9.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
+8 -8
View File
@@ -108,7 +108,7 @@
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<version>2.9.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
@@ -124,7 +124,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.2.4</version>
<version>3.2.3</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -156,7 +156,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.4</version>
<version>3.2.3</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -224,8 +224,8 @@
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>commons-beanutils</artifactId>
@@ -244,7 +244,7 @@
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-reload4j</artifactId>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
@@ -317,7 +317,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
<seaweedfs.client.version>3.33</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
<seaweedfs.client.version>3.13</seaweedfs.client.version>
<hadoop.version>3.2.3</hadoop.version>
</properties>
</project>
+2 -2
View File
@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>3.33</seaweedfs.client.version>
<seaweedfs.client.version>3.13</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
@@ -129,7 +129,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<version>2.9.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
@@ -91,7 +91,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -100,7 +100,7 @@
"step": 60
},
{
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -181,7 +181,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -190,7 +190,7 @@
"step": 60
},
{
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -277,7 +277,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -286,7 +286,7 @@
"step": 60
},
{
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -386,7 +386,7 @@
"steppedLine": false,
"targets": [
{
"expr": "rate(SeaweedFS_filerStore_request_total[1m]) * 5",
"expr": "rate(SeaweedFS_filer_request_total[1m]) * 5",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{type}}",
@@ -1929,4 +1929,4 @@
"timezone": "browser",
"title": "SeaweedFS ks8 heartbeat",
"version": 1
}
}
+1 -1
View File
@@ -59,7 +59,7 @@ func walkMetaFile(dst *os.File) error {
}
fmt.Fprintf(os.Stdout, "file %s %v\n", util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name), fullEntry.Entry.Attributes.String())
for i, chunk := range fullEntry.Entry.GetChunks() {
for i, chunk := range fullEntry.Entry.Chunks {
fmt.Fprintf(os.Stdout, " chunk: %d %v %d,%x%08x\n", i+1, chunk, chunk.Fid.VolumeId, chunk.Fid.FileKey, chunk.Fid.Cookie)
}
@@ -5,14 +5,13 @@ import (
"errors"
"flag"
"fmt"
"io"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"io"
)
var (
@@ -30,7 +29,7 @@ func main() {
vid := uint32(*volumeId)
eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
fmt.Printf("%d,%x%08x %d %v %d %x\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob), resp.NeedleBlobCompressed, resp.LastModified, resp.Crc)
fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob))
return nil
}
+11 -11
View File
@@ -15,44 +15,44 @@ clean:
debug_shell:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- shell
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- shell
debug_mount:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000
debug_server:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
debug_volume:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- volume -dir=~/tmp/100 -port 8564 -max=30 -preStopSeconds=2
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- volume -dir=~/tmp/100 -port 8564 -max=30 -preStopSeconds=2
debug_webdav:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 webdav
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 webdav
debug_s3:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 s3
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 s3
debug_filer_copy:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h
debug_filer_remote_sync_dir:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.remote.sync -filer="localhost:8888" -dir=/buckets/b2 -timeAgo=1h
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -dir=/buckets/b2 -timeAgo=1h
debug_filer_remote_sync_buckets:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.remote.sync -filer="localhost:8888" -createBucketAt=cloud1 -timeAgo=1h
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -createBucketAt=cloud1 -timeAgo=1h
debug_master_follower:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 master.follower
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 master.follower
debug_filer_sync:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.sync -a 192.168.2.7:8888 -b 192.168.2.7:8889 -isActivePassive -b.debug
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.sync -a 192.168.2.7:8888 -b 192.168.2.7:8889 -isActivePassive -b.debug
+2 -3
View File
@@ -2,7 +2,6 @@ package command
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
@@ -114,7 +113,7 @@ func runBackup(cmd *Command, args []string) bool {
return true
}
}
v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
if err != nil {
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
return true
@@ -139,7 +138,7 @@ func runBackup(cmd *Command, args []string) bool {
// remove the old data
v.Destroy()
// recreate an empty volume
v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
if err != nil {
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
return true
+24 -1
View File
@@ -41,6 +41,7 @@ type BenchmarkOptions struct {
grpcDialOption grpc.DialOption
masterClient *wdclient.MasterClient
fsync *bool
useTcp *bool
}
var (
@@ -67,6 +68,7 @@ func init() {
b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
b.fsync = cmdBenchmark.Flag.Bool("fsync", false, "flush data to disk after write")
b.useTcp = cmdBenchmark.Flag.Bool("useTcp", false, "send data via tcp")
sharedBytes = make([]byte, 1024)
}
@@ -223,6 +225,8 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
random := rand.New(rand.NewSource(time.Now().UnixNano()))
volumeTcpClient := wdclient.NewVolumeTcpClient()
for id := range idChan {
start := time.Now()
fileSize := int64(*b.fileSize + random.Intn(64))
@@ -243,7 +247,15 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
if !isSecure && assignResult.Auth != "" {
isSecure = true
}
if _, err := fp.Upload(0, b.masterClient.GetMaster, false, assignResult.Auth, b.grpcDialOption); err == nil {
if *b.useTcp {
if uploadByTcp(volumeTcpClient, fp) {
fileIdLineChan <- fp.Fid
s.completed++
s.transferred += fileSize
} else {
s.failed++
}
} else if _, err := fp.Upload(0, b.masterClient.GetMaster, false, assignResult.Auth, b.grpcDialOption); err == nil {
if random.Intn(100) < *b.deletePercentage {
s.total++
delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
@@ -329,6 +341,17 @@ func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan b
}
}
func uploadByTcp(volumeTcpClient *wdclient.VolumeTcpClient, fp *operation.FilePart) bool {
err := volumeTcpClient.PutFileChunk(fp.Server, fp.Fid, uint32(fp.FileSize), fp.Reader)
if err != nil {
glog.Errorf("upload chunk err: %v", err)
return false
}
return true
}
func readFileIds(fileName string, fileIdLineChan chan string) {
file, err := os.Open(fileName) // For read access.
if err != nil {
+1 -1
View File
@@ -41,7 +41,7 @@ func runCompact(cmd *Command, args []string) bool {
preallocate := *compactVolumePreallocate * (1 << 20)
vid := needle.VolumeId(*compactVolumeId)
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, 0, 0)
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, 0)
if err != nil {
glog.Fatalf("Load Volume [ERROR] %s\n", err)
}
+2 -3
View File
@@ -113,7 +113,6 @@ func init() {
filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
// start iam on filer
filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
@@ -163,7 +162,7 @@ func runFiler(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort)
go stats_collect.StartMetricsServer(*f.metricsHttpPort)
filerAddress := util.JoinHostPort(*f.ip, *f.port)
startDelay := time.Duration(2)
@@ -299,7 +298,7 @@ func (fo *FilerOptions) startFiler() {
localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
}
if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
glog.Fatalf("Failed to remove %s, error: %v", localSocket, err)
}
go func() {
// start on local unix socket
-15
View File
@@ -19,7 +19,6 @@ type FilerBackupOptions struct {
debug *bool
proxyByFiler *bool
timeAgo *time.Duration
retentionDays *int
}
var (
@@ -34,8 +33,6 @@ func init() {
filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers")
filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files")
filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
filerBackupOptions.retentionDays = cmdFilerBackup.Flag.Int("retentionDays", 0, "incremental backup retention days")
}
var cmdFilerBackup = &Command{
@@ -126,18 +123,6 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs)
})
if dataSink.IsIncremental() && *filerBackupOptions.retentionDays > 0 {
go func() {
for {
now := time.Now()
time.Sleep(time.Hour * 24)
key := util.Join(targetPath, now.Add(-1*time.Hour*24*time.Duration(*filerBackupOptions.retentionDays)).Format("2006-01-02"))
_ = dataSink.DeleteEntry(util.Join(targetPath, key), true, true, nil)
glog.V(0).Infof("incremental backup delete directory:%s", key)
}
}()
}
return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, clientEpoch, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
}
+1 -1
View File
@@ -114,7 +114,7 @@ func runFilerCat(cmd *Command, args []string) bool {
filerCat.filerClient = client
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.GetChunks(), 0, int64(filer.FileSize(respLookupEntry.Entry)))
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry)))
})
+4 -4
View File
@@ -365,7 +365,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
if flushErr != nil {
return flushErr
}
chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0, time.Now().UnixNano()))
chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0))
}
if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
@@ -450,7 +450,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
uploadError = fmt.Errorf("upload %v result: %v\n", fileName, uploadResult.Error)
return
}
chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize, time.Now().UnixNano())
chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize)
fmt.Printf("uploaded %s-%d [%d,%d)\n", fileName, i+1, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
}(i)
@@ -530,7 +530,7 @@ func detectMimeType(f *os.File) string {
return mimeType
}
func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error) {
func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, err error) {
finalFileId, uploadResult, flushErr, _ := operation.UploadWithRetry(
worker,
@@ -561,7 +561,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
if uploadResult.Error != "" {
return nil, fmt.Errorf("upload result: %v", uploadResult.Error)
}
return uploadResult.ToPbFileChunk(finalFileId, offset, tsNs), nil
return uploadResult.ToPbFileChunk(finalFileId, offset), nil
}
var _ = filer_pb.FilerClient(&FileCopyWorker{})
+4 -9
View File
@@ -37,16 +37,11 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
var lastLogTsNs = time.Now().UnixNano()
processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
if resp.EventNotification.NewEntry != nil {
if *option.storageClass == "" {
if _, ok := resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass]; ok {
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
}
} else {
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
}
if *option.storageClass == "" {
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
} else {
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
}
processor.AddSyncJob(resp)
return nil
}, 3*time.Second, func(counter int64, lastTsNs int64) error {
+4 -18
View File
@@ -43,7 +43,6 @@ type SyncOptions struct {
bFromTsMs *int64
aProxyByFiler *bool
bProxyByFiler *bool
metricsHttpIp *string
metricsHttpPort *int
concurrency *int
clientId int32
@@ -87,7 +86,6 @@ func init() {
syncOptions.concurrency = cmdFilerSynchronize.Flag.Int("concurrency", DefaultConcurrencyLimit, "The maximum number of files that will be synced concurrently.")
syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file")
syncMemProfile = cmdFilerSynchronize.Flag.String("memprofile", "", "memory profile output file")
syncOptions.metricsHttpIp = cmdFilerSynchronize.Flag.String("metricsIp", "", "metrics listen ip")
syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
syncOptions.clientId = util.RandomInt32()
}
@@ -121,7 +119,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
filerB := pb.ServerAddress(*syncOptions.filerB)
// start filer.sync metrics server
go statsCollect.StartMetricsServer(*syncOptions.metricsHttpIp, *syncOptions.metricsHttpPort)
go statsCollect.StartMetricsServer(*syncOptions.metricsHttpPort)
// read a filer signature
aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
@@ -399,11 +397,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
return nil
}
key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
return fmt.Errorf("create entry1 : %v", err)
} else {
return nil
}
return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
}
// this is something special?
@@ -431,11 +425,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
}
// create the new entry
newKey := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
if err := dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures); err != nil {
return fmt.Errorf("create entry2 : %v", err)
} else {
return nil
}
return dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures)
} else {
// new key is outside of the watched directory
@@ -449,11 +439,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
if strings.HasPrefix(string(sourceNewKey), sourcePath) {
// new key is in the watched directory
key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
return fmt.Errorf("create entry3 : %v", err)
} else {
return nil
}
return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
} else {
// new key is also outside of the watched directory
// skip
+2 -13
View File
@@ -31,7 +31,6 @@ var cmdFix = &Command{
var (
fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
fixIgnoreError = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
)
type VolumeFileScanner4Fix struct {
@@ -127,21 +126,11 @@ func doFixOneVolume(basepath string, baseFileName string, collection string, vol
}
if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
err := fmt.Errorf("scan .dat File: %v", err)
if *fixIgnoreError {
glog.Error(err)
} else {
glog.Fatal(err)
}
glog.Fatalf("scan .dat File: %v", err)
}
if err := nm.SaveToIdx(indexFileName); err != nil {
os.Remove(indexFileName)
err := fmt.Errorf("save to .idx File: %v", err)
if *fixIgnoreError {
glog.Error(err)
} else {
glog.Fatal(err)
}
glog.Fatalf("save to .idx File: %v", err)
}
}
+1 -10
View File
@@ -47,10 +47,7 @@ func runFuse(cmd *Command, args []string) bool {
for i++; i < rawArgsLen && rawArgs[i] != ' '; i++ {
option.WriteByte(rawArgs[i])
}
// ignore "-o"
if option.String() != "o" {
options = append(options, parameter{option.String(), "true"})
}
options = append(options, parameter{option.String(), "true"})
option.Reset()
// equal separator start option with pending value
@@ -193,12 +190,6 @@ func runFuse(cmd *Command, args []string) bool {
}
case "fusermount.path":
fusermountPath = parameter.value
default:
t := parameter.name
if parameter.value != "true" {
t = fmt.Sprintf("%s=%s", parameter.name, parameter.value)
}
mountOptions.extraOptions = append(mountOptions.extraOptions, t)
}
}
+1 -9
View File
@@ -2,7 +2,6 @@ package command
import (
"fmt"
hashicorpRaft "github.com/hashicorp/raft"
"net/http"
"os"
"path"
@@ -119,7 +118,7 @@ func runMaster(cmd *Command, args []string) bool {
glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
}
go stats_collect.StartMetricsServer(*m.ipBind, *m.metricsHttpPort)
go stats_collect.StartMetricsServer(*m.metricsHttpPort)
startMaster(m, masterWhiteList)
return true
@@ -254,13 +253,6 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
go httpS.Serve(masterListener)
}
grace.OnInterrupt(ms.Shutdown)
grace.OnInterrupt(grpcS.Stop)
grace.OnReload(func() {
if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
ms.Topo.HashicorpRaft.LeadershipTransfer()
}
})
select {}
}
-1
View File
@@ -31,7 +31,6 @@ type MountOptions struct {
debugPort *int
localSocket *string
disableXAttr *bool
extraOptions []string
}
var (
+2 -3
View File
@@ -166,7 +166,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
// mount fuse
fuseMountOptions := &fuse.MountOptions{
AllowOther: *option.allowOthers,
Options: option.extraOptions,
Options: nil,
MaxBackground: 128,
MaxWrite: 1024 * 1024 * 2,
MaxReadAhead: 1024 * 1024 * 2,
@@ -256,8 +256,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
seaweedFileSystem.StartBackgroundTasks()
glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
glog.V(0).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
server.Serve()
+1 -1
View File
@@ -144,7 +144,7 @@ func runS3(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
go stats_collect.StartMetricsServer(*s3StandaloneOptions.bindIp, *s3StandaloneOptions.metricsHttpPort)
go stats_collect.StartMetricsServer(*s3StandaloneOptions.metricsHttpPort)
return s3StandaloneOptions.startS3Server()
+15 -15
View File
@@ -41,13 +41,13 @@ enabled = false
dbFile = "./filer.db" # sqlite db file
[mysql] # or memsql, tidb
# CREATE TABLE IF NOT EXISTS `filemeta` (
# `dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
# `name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
# `directory` TEXT NOT NULL COMMENT 'full path to parent directory',
# `meta` LONGBLOB,
# PRIMARY KEY (`dirhash`, `name`)
# ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) BINARY COMMENT 'directory or file name',
# directory TEXT BINARY COMMENT 'full path to parent directory',
# meta LONGBLOB,
# PRIMARY KEY (dirhash, name)
# ) DEFAULT CHARSET=utf8;
enabled = false
hostname = "localhost"
@@ -61,18 +61,18 @@ connection_max_lifetime_seconds = 0
interpolateParams = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
[mysql2] # or memsql, tidb
enabled = false
createTable = """
CREATE TABLE IF NOT EXISTS `%s` (
`dirhash` BIGINT NOT NULL,
`name` VARCHAR(766) NOT NULL,
`directory` TEXT NOT NULL,
`meta` LONGBLOB,
PRIMARY KEY (`dirhash`, `name`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
dirhash BIGINT,
name VARCHAR(1000) BINARY,
directory TEXT BINARY,
meta LONGBLOB,
PRIMARY KEY (dirhash, name)
) DEFAULT CHARSET=utf8;
"""
hostname = "localhost"
port = 3306
@@ -85,7 +85,7 @@ connection_max_lifetime_seconds = 0
interpolateParams = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
[postgres] # or cockroachdb, YugabyteDB
# CREATE TABLE IF NOT EXISTS filemeta (
+6 -9
View File
@@ -92,11 +92,10 @@ func init() {
masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server")
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
@@ -125,7 +124,6 @@ func init() {
serverOptions.v.readMode = cmdServer.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
serverOptions.v.ldbTimeout = cmdServer.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 64, "limit total concurrent download size")
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
@@ -133,7 +131,7 @@ func init() {
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", false, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally")
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
@@ -156,7 +154,6 @@ func init() {
webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
webdavOptions.filerRootPath = cmdServer.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port")
@@ -234,7 +231,7 @@ func runServer(cmd *Command, args []string) bool {
webdavOptions.filer = &filerAddress
mqBrokerOptions.filerGroup = filerOptions.filerGroup
go stats_collect.StartMetricsServer(*serverBindIp, *serverMetricsHttpPort)
go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
folders := strings.Split(*volumeDataFolders, ",")
+2 -7
View File
@@ -68,7 +68,6 @@ type VolumeServerOptions struct {
inflightUploadDataTimeout *time.Duration
hasSlowRead *bool
readBufferSizeMB *int
ldbTimeout *int64
}
func init() {
@@ -93,14 +92,13 @@ func init() {
v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
v.ldbTimeout = cmdVolume.Flag.Int64("index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 256, "limit total concurrent upload size")
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 256, "limit total concurrent download size")
v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", false, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
}
@@ -130,7 +128,7 @@ func runVolume(cmd *Command, args []string) bool {
grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
}
go stats_collect.StartMetricsServer(*v.bindIp, *v.metricsHttpPort)
go stats_collect.StartMetricsServer(*v.metricsHttpPort)
minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
@@ -251,7 +249,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
*v.inflightUploadDataTimeout,
*v.hasSlowRead,
*v.readBufferSizeMB,
*v.ldbTimeout,
)
// starting grpc server
grpcS := v.startGrpcService(volumeServer)
@@ -268,8 +265,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
// starting the cluster http server
clusterHttpServer := v.startClusterHttpService(volumeMux)
grace.OnReload(volumeServer.LoadNewVolumes)
stopChan := make(chan bool)
grace.OnInterrupt(func() {
fmt.Println("volume server has been killed")
+1 -4
View File
@@ -13,7 +13,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/util"
)
@@ -23,7 +23,6 @@ var (
type WebDavOption struct {
filer *string
filerRootPath *string
port *int
collection *string
replication *string
@@ -45,7 +44,6 @@ func init() {
webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB")
webDavStandaloneOptions.filerRootPath = cmdWebDav.Flag.String("filer.path", "/", "use this remote path from filer server")
}
var cmdWebDav = &Command{
@@ -106,7 +104,6 @@ func (wo *WebDavOption) startWebDav() bool {
ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
Filer: filerAddress,
FilerRootPath: *wo.filerRootPath,
GrpcDialOption: grpcDialOption,
Collection: *wo.collection,
Replication: *wo.replication,
@@ -158,7 +158,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
meta = util.MaybeGzipData(meta)
}
+2 -2
View File
@@ -157,7 +157,7 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry)
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
meta = util.MaybeGzipData(meta)
}
model := &Model{
@@ -196,7 +196,7 @@ func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry)
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
meta = util.MaybeGzipData(meta)
}
model := &Model{
+1 -1
View File
@@ -100,7 +100,7 @@ func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer.Entry
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
meta = util.MaybeGzipData(meta)
}
+1 -6
View File
@@ -33,7 +33,7 @@ func (f *Filer) LoadConfiguration(config *util.ViperProxy) (isFresh bool) {
if !hasDefaultStoreConfigured {
println()
println("Supported filer stores are the following. If not found, check the full version.")
println("Supported filer stores are:")
for _, store := range Stores {
println(" " + store.GetName())
}
@@ -63,11 +63,6 @@ func (f *Filer) LoadConfiguration(config *util.ViperProxy) (isFresh bool) {
if !found {
continue
}
if !config.GetBool(key + ".enabled") {
continue
}
store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(FilerStore)
if err := store.Initialize(config, key+"."); err != nil {
glog.Fatalf("Failed to initialize store for %s: %+v", key, err)
+2 -6
View File
@@ -46,7 +46,7 @@ type Entry struct {
}
func (entry *Entry) Size() uint64 {
return maxUint64(maxUint64(TotalSize(entry.GetChunks()), entry.FileSize), uint64(len(entry.Content)))
return maxUint64(maxUint64(TotalSize(entry.Chunks), entry.FileSize), uint64(len(entry.Content)))
}
func (entry *Entry) Timestamp() time.Time {
@@ -91,7 +91,7 @@ func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry) {
}
message.IsDirectory = entry.IsDirectory()
message.Attributes = EntryAttributeToPb(entry)
message.Chunks = entry.GetChunks()
message.Chunks = entry.Chunks
message.Extended = entry.Extended
message.HardLinkId = entry.HardLinkId
message.HardLinkCounter = entry.HardLinkCounter
@@ -123,10 +123,6 @@ func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
}
}
func (entry *Entry) GetChunks() []*filer_pb.FileChunk {
return entry.Chunks
}
func FromPbEntry(dir string, entry *filer_pb.Entry) *Entry {
t := &Entry{}
t.FullPath = util.NewFullPath(dir, entry.Name)
+1 -1
View File
@@ -82,7 +82,7 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (er
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
}
if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
meta = weed_util.MaybeGzipData(meta)
}
-153
View File
@@ -1,153 +0,0 @@
package filer
import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"sync"
)
type ChunkGroup struct {
lookupFn wdclient.LookupFileIdFunctionType
chunkCache chunk_cache.ChunkCache
sections map[SectionIndex]*FileChunkSection
sectionsLock sync.RWMutex
}
func NewChunkGroup(lookupFn wdclient.LookupFileIdFunctionType, chunkCache chunk_cache.ChunkCache, chunks []*filer_pb.FileChunk) (*ChunkGroup, error) {
group := &ChunkGroup{
lookupFn: lookupFn,
chunkCache: chunkCache,
sections: make(map[SectionIndex]*FileChunkSection),
}
err := group.SetChunks(chunks)
return group, err
}
func (group *ChunkGroup) AddChunk(chunk *filer_pb.FileChunk) error {
group.sectionsLock.Lock()
defer group.sectionsLock.Unlock()
sectionIndexStart, sectionIndexStop := SectionIndex(chunk.Offset/SectionSize), SectionIndex((chunk.Offset+int64(chunk.Size))/SectionSize)
for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
section, found := group.sections[si]
if !found {
section = NewFileChunkSection(si)
group.sections[si] = section
}
section.addChunk(chunk)
}
return nil
}
func (group *ChunkGroup) ReadDataAt(fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
group.sectionsLock.RLock()
defer group.sectionsLock.RUnlock()
sectionIndexStart, sectionIndexStop := SectionIndex(offset/SectionSize), SectionIndex((offset+int64(len(buff)))/SectionSize)
for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
section, found := group.sections[si]
rangeStart, rangeStop := max(offset, int64(si*SectionSize)), min(offset+int64(len(buff)), int64((si+1)*SectionSize))
if !found {
for i := rangeStart; i < rangeStop; i++ {
buff[i-offset] = 0
}
continue
}
xn, xTsNs, xErr := section.readDataAt(group, fileSize, buff[rangeStart-offset:rangeStop-offset], rangeStart)
if xErr != nil {
err = xErr
}
n += xn
tsNs = max(tsNs, xTsNs)
}
return
}
func (group *ChunkGroup) SetChunks(chunks []*filer_pb.FileChunk) error {
group.sectionsLock.RLock()
defer group.sectionsLock.RUnlock()
var dataChunks []*filer_pb.FileChunk
for _, chunk := range chunks {
if !chunk.IsChunkManifest {
dataChunks = append(dataChunks, chunk)
continue
}
resolvedChunks, err := ResolveOneChunkManifest(group.lookupFn, chunk)
if err != nil {
return err
}
dataChunks = append(dataChunks, resolvedChunks...)
}
sections := make(map[SectionIndex]*FileChunkSection)
for _, chunk := range dataChunks {
sectionIndexStart, sectionIndexStop := SectionIndex(chunk.Offset/SectionSize), SectionIndex((chunk.Offset+int64(chunk.Size))/SectionSize)
for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
section, found := sections[si]
if !found {
section = NewFileChunkSection(si)
sections[si] = section
}
section.chunks = append(section.chunks, chunk)
}
}
group.sections = sections
return nil
}
const (
// see weedfs_file_lseek.go
SEEK_DATA uint32 = 3 // seek to next data after the offset
// SEEK_HOLE uint32 = 4 // seek to next hole after the offset
)
// FIXME: needa tests
func (group *ChunkGroup) SearchChunks(offset, fileSize int64, whence uint32) (found bool, out int64) {
group.sectionsLock.RLock()
defer group.sectionsLock.RUnlock()
return group.doSearchChunks(offset, fileSize, whence)
}
func (group *ChunkGroup) doSearchChunks(offset, fileSize int64, whence uint32) (found bool, out int64) {
sectionIndex, maxSectionIndex := SectionIndex(offset/SectionSize), SectionIndex(fileSize/SectionSize)
if whence == SEEK_DATA {
for si := sectionIndex; si < maxSectionIndex+1; si++ {
section, foundSection := group.sections[si]
if !foundSection {
continue
}
sectionStart := section.DataStartOffset(group, offset, fileSize)
if sectionStart == -1 {
continue
}
return true, sectionStart
}
return false, 0
} else {
// whence == SEEK_HOLE
for si := sectionIndex; si < maxSectionIndex; si++ {
section, foundSection := group.sections[si]
if !foundSection {
return true, offset
}
holeStart := section.NextStopOffset(group, offset, fileSize)
if holeStart%SectionSize == 0 {
continue
}
return true, holeStart
}
return true, fileSize
}
}
-36
View File
@@ -1,36 +0,0 @@
package filer
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestChunkGroup_doSearchChunks(t *testing.T) {
type fields struct {
sections map[SectionIndex]*FileChunkSection
}
type args struct {
offset int64
fileSize int64
whence uint32
}
tests := []struct {
name string
fields fields
args args
wantFound bool
wantOut int64
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
group := &ChunkGroup{
sections: tt.fields.sections,
}
gotFound, gotOut := group.doSearchChunks(tt.args.offset, tt.args.fileSize, tt.args.whence)
assert.Equalf(t, tt.wantFound, gotFound, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence)
assert.Equalf(t, tt.wantOut, gotOut, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence)
})
}
}
+2 -2
View File
@@ -264,7 +264,7 @@ func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer
}
}
manifestChunk, err = saveFunc(bytes.NewReader(data), "", 0, 0)
manifestChunk, err = saveFunc(bytes.NewReader(data), "", 0)
if err != nil {
return nil, err
}
@@ -275,4 +275,4 @@ func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer
return
}
type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error)
type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, err error)
-134
View File
@@ -1,134 +0,0 @@
package filer
import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"sync"
)
const SectionSize = 2 * 1024 * 1024 * 32 // 64MiB
type SectionIndex int64
type FileChunkSection struct {
sectionIndex SectionIndex
chunks []*filer_pb.FileChunk
visibleIntervals *IntervalList[*VisibleInterval]
chunkViews *IntervalList[*ChunkView]
reader *ChunkReadAt
lock sync.Mutex
}
func NewFileChunkSection(si SectionIndex) *FileChunkSection {
return &FileChunkSection{
sectionIndex: si,
}
}
func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error {
section.lock.Lock()
defer section.lock.Unlock()
start, stop := max(int64(section.sectionIndex)*SectionSize, chunk.Offset), min(((int64(section.sectionIndex)+1)*SectionSize), chunk.Offset+int64(chunk.Size))
section.chunks = append(section.chunks, chunk)
if section.visibleIntervals == nil {
section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
} else {
MergeIntoVisibles(section.visibleIntervals, start, stop, chunk)
garbageFileIds := FindGarbageChunks(section.visibleIntervals, start, stop)
removeGarbageChunks(section, garbageFileIds)
}
if section.chunkViews != nil {
MergeIntoChunkViews(section.chunkViews, start, stop, chunk)
}
return nil
}
func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]struct{}) {
for i := 0; i < len(section.chunks); {
t := section.chunks[i]
length := len(section.chunks)
if _, found := garbageFileIds[t.FileId]; found {
if i < length-1 {
section.chunks[i] = section.chunks[length-1]
}
section.chunks = section.chunks[:length-1]
} else {
i++
}
}
}
func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64) {
if section.visibleIntervals == nil {
section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks)
if section.reader != nil {
_ = section.reader.Close()
section.reader = nil
}
}
if section.chunkViews == nil {
section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
}
if section.reader == nil {
section.reader = NewChunkReaderAtFromClient(group.lookupFn, section.chunkViews, group.chunkCache, min(int64(section.sectionIndex+1)*SectionSize, fileSize))
}
section.reader.fileSize = fileSize
}
func (section *FileChunkSection) readDataAt(group *ChunkGroup, fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
section.lock.Lock()
defer section.lock.Unlock()
section.setupForRead(group, fileSize)
return section.reader.ReadAtWithTime(buff, offset)
}
func (section *FileChunkSection) DataStartOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
section.lock.Lock()
defer section.lock.Unlock()
section.setupForRead(group, fileSize)
for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
visible := x.Value
if visible.stop <= offset {
continue
}
if offset < visible.start {
return offset
}
return offset
}
return -1
}
func (section *FileChunkSection) NextStopOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
section.lock.Lock()
defer section.lock.Unlock()
section.setupForRead(group, fileSize)
isAfterOffset := false
for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
visible := x.Value
if !isAfterOffset {
if visible.stop <= offset {
continue
}
isAfterOffset = true
}
if offset < visible.start {
return offset
}
// now visible.start <= offset
if offset < visible.stop {
offset = visible.stop
}
}
return offset
}
-48
View File
@@ -1,48 +0,0 @@
package filer
import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"testing"
)
func Test_removeGarbageChunks(t *testing.T) {
section := NewFileChunkSection(0)
section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "0",
Offset: 0,
Size: 1,
ModifiedTsNs: 0,
})
section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "1",
Offset: 1,
Size: 1,
ModifiedTsNs: 1,
})
section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "2",
Offset: 2,
Size: 1,
ModifiedTsNs: 2,
})
section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "3",
Offset: 3,
Size: 1,
ModifiedTsNs: 3,
})
section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "4",
Offset: 4,
Size: 1,
ModifiedTsNs: 4,
})
garbageFileIds := make(map[string]struct{})
garbageFileIds["0"] = struct{}{}
garbageFileIds["2"] = struct{}{}
garbageFileIds["4"] = struct{}{}
removeGarbageChunks(section, garbageFileIds)
if len(section.chunks) != 2 {
t.Errorf("remove chunk 2 failed")
}
}
+137 -124
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"golang.org/x/exp/slices"
"math"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -30,22 +31,19 @@ func FileSize(entry *filer_pb.Entry) (size uint64) {
fileSize = maxUint64(fileSize, uint64(entry.RemoteEntry.RemoteSize))
}
}
return maxUint64(TotalSize(entry.GetChunks()), fileSize)
return maxUint64(TotalSize(entry.Chunks), fileSize)
}
func ETag(entry *filer_pb.Entry) (etag string) {
if entry.Attributes == nil || entry.Attributes.Md5 == nil {
return ETagChunks(entry.GetChunks())
return ETagChunks(entry.Chunks)
}
return fmt.Sprintf("%x", entry.Attributes.Md5)
}
func ETagEntry(entry *Entry) (etag string) {
if entry.IsInRemoteOnly() {
return entry.Remote.RemoteETag
}
if entry.Attr.Md5 == nil {
return ETagChunks(entry.GetChunks())
return ETagChunks(entry.Chunks)
}
return fmt.Sprintf("%x", entry.Attr.Md5)
}
@@ -65,15 +63,8 @@ func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks
visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, 0, math.MaxInt64)
compacted, garbage = SeparateGarbageChunks(visibles, chunks)
return
}
func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*filer_pb.FileChunk) (compacted []*filer_pb.FileChunk, garbage []*filer_pb.FileChunk) {
fileIds := make(map[string]bool)
for x := visibles.Front(); x != nil; x = x.Next {
interval := x.Value
for _, interval := range visibles {
fileIds[interval.fileId] = true
}
for _, chunk := range chunks {
@@ -83,18 +74,7 @@ func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*f
garbage = append(garbage, chunk)
}
}
return compacted, garbage
}
func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, stop int64) (garbageFileIds map[string]struct{}) {
garbageFileIds = make(map[string]struct{})
for x := visibles.Front(); x != nil; x = x.Next {
interval := x.Value
offset := interval.start - interval.offsetInChunk
if start <= offset && offset+int64(interval.chunkSize) <= stop {
garbageFileIds[interval.fileId] = struct{}{}
}
}
return
}
@@ -148,39 +128,20 @@ func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_p
}
type ChunkView struct {
FileId string
OffsetInChunk int64 // offset within the chunk
ViewSize uint64
ViewOffset int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
ChunkSize uint64
CipherKey []byte
IsGzipped bool
ModifiedTsNs int64
}
func (cv *ChunkView) SetStartStop(start, stop int64) {
cv.OffsetInChunk += start - cv.ViewOffset
cv.ViewOffset = start
cv.ViewSize = uint64(stop - start)
}
func (cv *ChunkView) Clone() IntervalValue {
return &ChunkView{
FileId: cv.FileId,
OffsetInChunk: cv.OffsetInChunk,
ViewSize: cv.ViewSize,
ViewOffset: cv.ViewOffset,
ChunkSize: cv.ChunkSize,
CipherKey: cv.CipherKey,
IsGzipped: cv.IsGzipped,
ModifiedTsNs: cv.ModifiedTsNs,
}
FileId string
Offset int64
Size uint64
LogicOffset int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
ChunkSize uint64
CipherKey []byte
IsGzipped bool
}
func (cv *ChunkView) IsFullChunk() bool {
return cv.ViewSize == cv.ChunkSize
return cv.Size == cv.ChunkSize
}
func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {
visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, offset, offset+size)
@@ -188,7 +149,7 @@ func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*
}
func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int64) (views []*ChunkView) {
stop := offset + size
if size == math.MaxInt64 {
@@ -198,112 +159,164 @@ func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset i
stop = math.MaxInt64
}
chunkViews = NewIntervalList[*ChunkView]()
for x := visibles.Front(); x != nil; x = x.Next {
chunk := x.Value
for _, chunk := range visibles {
chunkStart, chunkStop := max(offset, chunk.start), min(stop, chunk.stop)
if chunkStart < chunkStop {
chunkView := &ChunkView{
FileId: chunk.fileId,
OffsetInChunk: chunkStart - chunk.start + chunk.offsetInChunk,
ViewSize: uint64(chunkStop - chunkStart),
ViewOffset: chunkStart,
ChunkSize: chunk.chunkSize,
CipherKey: chunk.cipherKey,
IsGzipped: chunk.isGzipped,
ModifiedTsNs: chunk.modifiedTsNs,
}
chunkViews.AppendInterval(&Interval[*ChunkView]{
StartOffset: chunkStart,
StopOffset: chunkStop,
TsNs: chunk.modifiedTsNs,
Value: chunkView,
Prev: nil,
Next: nil,
views = append(views, &ChunkView{
FileId: chunk.fileId,
Offset: chunkStart - chunk.start + chunk.chunkOffset,
Size: uint64(chunkStop - chunkStart),
LogicOffset: chunkStart,
ChunkSize: chunk.chunkSize,
CipherKey: chunk.cipherKey,
IsGzipped: chunk.isGzipped,
})
}
}
return chunkViews
return views
}
func MergeIntoVisibles(visibles *IntervalList[*VisibleInterval], start int64, stop int64, chunk *filer_pb.FileChunk) {
func logPrintf(name string, visibles []VisibleInterval) {
newV := &VisibleInterval{
start: start,
stop: stop,
fileId: chunk.GetFileIdString(),
modifiedTsNs: chunk.ModifiedTsNs,
offsetInChunk: start - chunk.Offset, // the starting position in the chunk
chunkSize: chunk.Size, // size of the chunk
cipherKey: chunk.CipherKey,
isGzipped: chunk.IsCompressed,
}
visibles.InsertInterval(start, stop, chunk.ModifiedTsNs, newV)
/*
glog.V(0).Infof("%s len %d", name, len(visibles))
for _, v := range visibles {
glog.V(0).Infof("%s: [%d,%d) %s %d", name, v.start, v.stop, v.fileId, v.chunkOffset)
}
*/
}
func MergeIntoChunkViews(chunkViews *IntervalList[*ChunkView], start int64, stop int64, chunk *filer_pb.FileChunk) {
func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (newVisibles []VisibleInterval) {
chunkView := &ChunkView{
FileId: chunk.GetFileIdString(),
OffsetInChunk: start - chunk.Offset,
ViewSize: uint64(stop - start),
ViewOffset: start,
ChunkSize: chunk.Size,
CipherKey: chunk.CipherKey,
IsGzipped: chunk.IsCompressed,
ModifiedTsNs: chunk.ModifiedTsNs,
newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, 0, chunk.Size, chunk.CipherKey, chunk.IsCompressed)
length := len(visibles)
if length == 0 {
return append(visibles, newV)
}
last := visibles[length-1]
if last.stop <= chunk.Offset {
return append(visibles, newV)
}
chunkViews.InsertInterval(start, stop, chunk.ModifiedTsNs, chunkView)
logPrintf(" before", visibles)
// glog.V(0).Infof("newVisibles %d adding chunk [%d,%d) %s size:%d", len(newVisibles), chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Size)
chunkStop := chunk.Offset + int64(chunk.Size)
for _, v := range visibles {
if v.start < chunk.Offset && chunk.Offset < v.stop {
t := newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, v.chunkOffset, v.chunkSize, v.cipherKey, v.isGzipped)
newVisibles = append(newVisibles, t)
// glog.V(0).Infof("visible %d [%d,%d) =1> [%d,%d)", i, v.start, v.stop, t.start, t.stop)
}
if v.start < chunkStop && chunkStop < v.stop {
t := newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, v.chunkOffset+(chunkStop-v.start), v.chunkSize, v.cipherKey, v.isGzipped)
newVisibles = append(newVisibles, t)
// glog.V(0).Infof("visible %d [%d,%d) =2> [%d,%d)", i, v.start, v.stop, t.start, t.stop)
}
if chunkStop <= v.start || v.stop <= chunk.Offset {
newVisibles = append(newVisibles, v)
// glog.V(0).Infof("visible %d [%d,%d) =3> [%d,%d)", i, v.start, v.stop, v.start, v.stop)
}
}
newVisibles = append(newVisibles, newV)
logPrintf(" append", newVisibles)
for i := len(newVisibles) - 1; i >= 0; i-- {
if i > 0 && newV.start < newVisibles[i-1].start {
newVisibles[i] = newVisibles[i-1]
} else {
newVisibles[i] = newV
break
}
}
logPrintf(" sorted", newVisibles)
return newVisibles
}
// NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory
// If the file chunk content is a chunk manifest
func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval], err error) {
func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) {
chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)
if err != nil {
return
}
visibles2 := readResolvedChunks(chunks, 0, math.MaxInt64)
visibles2 := readResolvedChunks(chunks)
return visibles2, err
if true {
return visibles2, err
}
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
if a.Mtime == b.Mtime {
filer_pb.EnsureFid(a)
filer_pb.EnsureFid(b)
if a.Fid == nil || b.Fid == nil {
return true
}
return a.Fid.FileKey < b.Fid.FileKey
}
return a.Mtime < b.Mtime
})
for _, chunk := range chunks {
// glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
visibles = MergeIntoVisibles(visibles, chunk)
logPrintf("add", visibles)
}
if len(visibles) != len(visibles2) {
fmt.Printf("different visibles size %d : %d\n", len(visibles), len(visibles2))
} else {
for i := 0; i < len(visibles); i++ {
checkDifference(visibles[i], visibles2[i])
}
}
return
}
func checkDifference(x, y VisibleInterval) {
if x.start != y.start ||
x.stop != y.stop ||
x.fileId != y.fileId ||
x.modifiedTime != y.modifiedTime {
fmt.Printf("different visible %+v : %+v\n", x, y)
}
}
// find non-overlapping visible intervals
// visible interval map to one file chunk
type VisibleInterval struct {
start int64
stop int64
modifiedTsNs int64
fileId string
offsetInChunk int64
chunkSize uint64
cipherKey []byte
isGzipped bool
start int64
stop int64
modifiedTime int64
fileId string
chunkOffset int64
chunkSize uint64
cipherKey []byte
isGzipped bool
}
func (v *VisibleInterval) SetStartStop(start, stop int64) {
v.offsetInChunk += start - v.start
v.start, v.stop = start, stop
}
func (v *VisibleInterval) Clone() IntervalValue {
return &VisibleInterval{
start: v.start,
stop: v.stop,
modifiedTsNs: v.modifiedTsNs,
fileId: v.fileId,
offsetInChunk: v.offsetInChunk,
chunkSize: v.chunkSize,
cipherKey: v.cipherKey,
isGzipped: v.isGzipped,
func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, chunkOffset int64, chunkSize uint64, cipherKey []byte, isGzipped bool) VisibleInterval {
return VisibleInterval{
start: start,
stop: stop,
fileId: fileId,
modifiedTime: modifiedTime,
chunkOffset: chunkOffset, // the starting position in the chunk
chunkSize: chunkSize,
cipherKey: cipherKey,
isGzipped: isGzipped,
}
}
+18 -18
View File
@@ -17,14 +17,14 @@ func TestDoMinusChunks(t *testing.T) {
// clusterA append a new line and then clusterB also append a new line
// clusterA append a new line again
chunksInA := []*filer_pb.FileChunk{
{Offset: 0, Size: 3, FileId: "11", ModifiedTsNs: 100},
{Offset: 3, Size: 3, FileId: "22", SourceFileId: "2", ModifiedTsNs: 200},
{Offset: 6, Size: 3, FileId: "33", ModifiedTsNs: 300},
{Offset: 0, Size: 3, FileId: "11", Mtime: 100},
{Offset: 3, Size: 3, FileId: "22", SourceFileId: "2", Mtime: 200},
{Offset: 6, Size: 3, FileId: "33", Mtime: 300},
}
chunksInB := []*filer_pb.FileChunk{
{Offset: 0, Size: 3, FileId: "1", SourceFileId: "11", ModifiedTsNs: 100},
{Offset: 3, Size: 3, FileId: "2", ModifiedTsNs: 200},
{Offset: 6, Size: 3, FileId: "3", SourceFileId: "33", ModifiedTsNs: 300},
{Offset: 0, Size: 3, FileId: "1", SourceFileId: "11", Mtime: 100},
{Offset: 3, Size: 3, FileId: "2", Mtime: 200},
{Offset: 6, Size: 3, FileId: "3", SourceFileId: "33", Mtime: 300},
}
// clusterB using command "echo 'content' > hello.txt" to overwrite file
@@ -50,17 +50,17 @@ func TestDoMinusChunks(t *testing.T) {
func TestCompactFileChunksRealCase(t *testing.T) {
chunks := []*filer_pb.FileChunk{
{FileId: "2,512f31f2c0700a", Offset: 0, Size: 25 - 0, ModifiedTsNs: 5320497},
{FileId: "6,512f2c2e24e9e8", Offset: 868352, Size: 917585 - 868352, ModifiedTsNs: 5320492},
{FileId: "7,514468dd5954ca", Offset: 884736, Size: 901120 - 884736, ModifiedTsNs: 5325928},
{FileId: "5,5144463173fe77", Offset: 917504, Size: 2297856 - 917504, ModifiedTsNs: 5325894},
{FileId: "4,51444c7ab54e2d", Offset: 2301952, Size: 2367488 - 2301952, ModifiedTsNs: 5325900},
{FileId: "4,514450e643ad22", Offset: 2371584, Size: 2420736 - 2371584, ModifiedTsNs: 5325904},
{FileId: "6,514456a5e9e4d7", Offset: 2449408, Size: 2490368 - 2449408, ModifiedTsNs: 5325910},
{FileId: "3,51444f8d53eebe", Offset: 2494464, Size: 2555904 - 2494464, ModifiedTsNs: 5325903},
{FileId: "4,5144578b097c7e", Offset: 2560000, Size: 2596864 - 2560000, ModifiedTsNs: 5325911},
{FileId: "3,51445500b6b4ac", Offset: 2637824, Size: 2678784 - 2637824, ModifiedTsNs: 5325909},
{FileId: "1,51446285e52a61", Offset: 2695168, Size: 2715648 - 2695168, ModifiedTsNs: 5325922},
{FileId: "2,512f31f2c0700a", Offset: 0, Size: 25 - 0, Mtime: 5320497},
{FileId: "6,512f2c2e24e9e8", Offset: 868352, Size: 917585 - 868352, Mtime: 5320492},
{FileId: "7,514468dd5954ca", Offset: 884736, Size: 901120 - 884736, Mtime: 5325928},
{FileId: "5,5144463173fe77", Offset: 917504, Size: 2297856 - 917504, Mtime: 5325894},
{FileId: "4,51444c7ab54e2d", Offset: 2301952, Size: 2367488 - 2301952, Mtime: 5325900},
{FileId: "4,514450e643ad22", Offset: 2371584, Size: 2420736 - 2371584, Mtime: 5325904},
{FileId: "6,514456a5e9e4d7", Offset: 2449408, Size: 2490368 - 2449408, Mtime: 5325910},
{FileId: "3,51444f8d53eebe", Offset: 2494464, Size: 2555904 - 2494464, Mtime: 5325903},
{FileId: "4,5144578b097c7e", Offset: 2560000, Size: 2596864 - 2560000, Mtime: 5325911},
{FileId: "3,51445500b6b4ac", Offset: 2637824, Size: 2678784 - 2637824, Mtime: 5325909},
{FileId: "1,51446285e52a61", Offset: 2695168, Size: 2715648 - 2695168, Mtime: 5325922},
}
printChunks("before", chunks)
@@ -75,7 +75,7 @@ func TestCompactFileChunksRealCase(t *testing.T) {
func printChunks(name string, chunks []*filer_pb.FileChunk) {
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
if a.Offset == b.Offset {
return a.ModifiedTsNs < b.ModifiedTsNs
return a.Mtime < b.Mtime
}
return a.Offset < b.Offset
})
+52 -58
View File
@@ -1,31 +1,23 @@
package filer
import (
"container/list"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"golang.org/x/exp/slices"
)
func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval]) {
func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
var points []*Point
for _, chunk := range chunks {
if chunk.IsChunkManifest {
println("This should not happen! A manifest chunk found:", chunk.GetFileIdString())
}
start, stop := max(chunk.Offset, startOffset), min(chunk.Offset+int64(chunk.Size), stopOffset)
if start >= stop {
continue
}
points = append(points, &Point{
x: chunk.Offset,
ts: chunk.ModifiedTsNs,
ts: chunk.Mtime,
chunk: chunk,
isStart: true,
})
points = append(points, &Point{
x: chunk.Offset + int64(chunk.Size),
ts: chunk.ModifiedTsNs,
ts: chunk.Mtime,
chunk: chunk,
isStart: false,
})
@@ -41,45 +33,40 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
})
var prevX int64
queue := list.New() // points with higher ts are at the tail
visibles = NewIntervalList[*VisibleInterval]()
var prevPoint *Point
var queue []*Point
for _, point := range points {
if queue.Len() > 0 {
prevPoint = queue.Back().Value.(*Point)
} else {
prevPoint = nil
}
if point.isStart {
if prevPoint != nil {
if point.x != prevX && prevPoint.ts < point.ts {
addToVisibles(visibles, prevX, prevPoint, point)
if len(queue) > 0 {
lastIndex := len(queue) - 1
lastPoint := queue[lastIndex]
if point.x != prevX && lastPoint.ts < point.ts {
visibles = addToVisibles(visibles, prevX, lastPoint, point)
prevX = point.x
}
}
// insert into queue
if prevPoint == nil || prevPoint.ts < point.ts {
queue.PushBack(point)
prevX = point.x
} else {
for e := queue.Front(); e != nil; e = e.Next() {
if e.Value.(*Point).ts > point.ts {
queue.InsertBefore(point, e)
break
for i := len(queue); i >= 0; i-- {
if i == 0 || queue[i-1].ts <= point.ts {
if i == len(queue) {
prevX = point.x
}
queue = addToQueue(queue, i, point)
break
}
}
} else {
isLast := true
for e := queue.Back(); e != nil; e = e.Prev() {
if e.Value.(*Point).ts == point.ts {
queue.Remove(e)
lastIndex := len(queue) - 1
index := lastIndex
var startPoint *Point
for ; index >= 0; index-- {
startPoint = queue[index]
if startPoint.ts == point.ts {
queue = removeFromQueue(queue, index)
break
}
isLast = false
}
if isLast && prevPoint != nil {
addToVisibles(visibles, prevX, prevPoint, point)
if index == lastIndex && startPoint != nil {
visibles = addToVisibles(visibles, prevX, startPoint, point)
prevX = point.x
}
}
@@ -88,30 +75,37 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
return
}
func addToVisibles(visibles *IntervalList[*VisibleInterval], prevX int64, startPoint *Point, point *Point) {
if prevX < point.x {
chunk := startPoint.chunk
visible := &VisibleInterval{
start: prevX,
stop: point.x,
fileId: chunk.GetFileIdString(),
modifiedTsNs: chunk.ModifiedTsNs,
offsetInChunk: prevX - chunk.Offset,
chunkSize: chunk.Size,
cipherKey: chunk.CipherKey,
isGzipped: chunk.IsCompressed,
}
appendVisibleInterfal(visibles, visible)
func removeFromQueue(queue []*Point, index int) []*Point {
for i := index; i < len(queue)-1; i++ {
queue[i] = queue[i+1]
}
queue = queue[:len(queue)-1]
return queue
}
func appendVisibleInterfal(visibles *IntervalList[*VisibleInterval], visible *VisibleInterval) {
visibles.AppendInterval(&Interval[*VisibleInterval]{
StartOffset: visible.start,
StopOffset: visible.stop,
TsNs: visible.modifiedTsNs,
Value: visible,
})
func addToQueue(queue []*Point, index int, point *Point) []*Point {
queue = append(queue, point)
for i := len(queue) - 1; i > index; i-- {
queue[i], queue[i-1] = queue[i-1], queue[i]
}
return queue
}
func addToVisibles(visibles []VisibleInterval, prevX int64, startPoint *Point, point *Point) []VisibleInterval {
if prevX < point.x {
chunk := startPoint.chunk
visibles = append(visibles, VisibleInterval{
start: prevX,
stop: point.x,
fileId: chunk.GetFileIdString(),
modifiedTime: chunk.Mtime,
chunkOffset: prevX - chunk.Offset,
chunkSize: chunk.Size,
cipherKey: chunk.CipherKey,
isGzipped: chunk.IsCompressed,
})
}
return visibles
}
type Point struct {
+92 -162
View File
@@ -3,7 +3,6 @@ package filer
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"math"
"math/rand"
"testing"
)
@@ -12,70 +11,41 @@ func TestReadResolvedChunks(t *testing.T) {
chunks := []*filer_pb.FileChunk{
{
FileId: "a",
Offset: 0,
Size: 100,
ModifiedTsNs: 1,
FileId: "a",
Offset: 0,
Size: 100,
Mtime: 1,
},
{
FileId: "b",
Offset: 50,
Size: 100,
ModifiedTsNs: 2,
FileId: "b",
Offset: 50,
Size: 100,
Mtime: 2,
},
{
FileId: "c",
Offset: 200,
Size: 50,
ModifiedTsNs: 3,
FileId: "c",
Offset: 200,
Size: 50,
Mtime: 3,
},
{
FileId: "d",
Offset: 250,
Size: 50,
ModifiedTsNs: 4,
FileId: "d",
Offset: 250,
Size: 50,
Mtime: 4,
},
{
FileId: "e",
Offset: 175,
Size: 100,
ModifiedTsNs: 5,
FileId: "e",
Offset: 175,
Size: 100,
Mtime: 5,
},
}
visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
visibles := readResolvedChunks(chunks)
fmt.Printf("resolved to %d visible intervales\n", visibles.Len())
for x := visibles.Front(); x != nil; x = x.Next {
visible := x.Value
fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
}
}
func TestReadResolvedChunks2(t *testing.T) {
chunks := []*filer_pb.FileChunk{
{
FileId: "c",
Offset: 200,
Size: 50,
ModifiedTsNs: 3,
},
{
FileId: "e",
Offset: 200,
Size: 25,
ModifiedTsNs: 5,
},
}
visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
fmt.Printf("resolved to %d visible intervales\n", visibles.Len())
for x := visibles.Front(); x != nil; x = x.Next {
visible := x.Value
fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
for _, visible := range visibles {
fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTime)
}
}
@@ -102,13 +72,12 @@ func TestRandomizedReadResolvedChunks(t *testing.T) {
chunks = append(chunks, randomWrite(array, start, size, ts))
}
visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
visibles := readResolvedChunks(chunks)
for x := visibles.Front(); x != nil; x = x.Next {
visible := x.Value
for _, visible := range visibles {
for i := visible.start; i < visible.stop; i++ {
if array[i] != visible.modifiedTsNs {
t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTsNs)
if array[i] != visible.modifiedTime {
t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTime)
}
}
}
@@ -123,10 +92,10 @@ func randomWrite(array []int64, start int64, size int64, ts int64) *filer_pb.Fil
}
// fmt.Printf("write [%d,%d) %d\n", start, start+size, ts)
return &filer_pb.FileChunk{
FileId: "",
Offset: start,
Size: uint64(size),
ModifiedTsNs: ts,
FileId: "",
Offset: start,
Size: uint64(size),
Mtime: ts,
}
}
@@ -136,16 +105,16 @@ func TestSequentialReadResolvedChunks(t *testing.T) {
var chunks []*filer_pb.FileChunk
for ts := int64(0); ts < 13; ts++ {
chunks = append(chunks, &filer_pb.FileChunk{
FileId: "",
Offset: chunkSize * ts,
Size: uint64(chunkSize),
ModifiedTsNs: 1,
FileId: "",
Offset: chunkSize * ts,
Size: uint64(chunkSize),
Mtime: 1,
})
}
visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
visibles := readResolvedChunks(chunks)
fmt.Printf("visibles %d", visibles.Len())
fmt.Printf("visibles %d", len(visibles))
}
@@ -153,128 +122,89 @@ func TestActualReadResolvedChunks(t *testing.T) {
chunks := []*filer_pb.FileChunk{
{
FileId: "5,e7b96fef48",
Offset: 0,
Size: 2097152,
ModifiedTsNs: 1634447487595823000,
FileId: "5,e7b96fef48",
Offset: 0,
Size: 2097152,
Mtime: 1634447487595823000,
},
{
FileId: "5,e5562640b9",
Offset: 2097152,
Size: 2097152,
ModifiedTsNs: 1634447487595826000,
FileId: "5,e5562640b9",
Offset: 2097152,
Size: 2097152,
Mtime: 1634447487595826000,
},
{
FileId: "5,df033e0fe4",
Offset: 4194304,
Size: 2097152,
ModifiedTsNs: 1634447487595827000,
FileId: "5,df033e0fe4",
Offset: 4194304,
Size: 2097152,
Mtime: 1634447487595827000,
},
{
FileId: "7,eb08148a9b",
Offset: 6291456,
Size: 2097152,
ModifiedTsNs: 1634447487595827000,
FileId: "7,eb08148a9b",
Offset: 6291456,
Size: 2097152,
Mtime: 1634447487595827000,
},
{
FileId: "7,e0f92d1604",
Offset: 8388608,
Size: 2097152,
ModifiedTsNs: 1634447487595828000,
FileId: "7,e0f92d1604",
Offset: 8388608,
Size: 2097152,
Mtime: 1634447487595828000,
},
{
FileId: "7,e33cb63262",
Offset: 10485760,
Size: 2097152,
ModifiedTsNs: 1634447487595828000,
FileId: "7,e33cb63262",
Offset: 10485760,
Size: 2097152,
Mtime: 1634447487595828000,
},
{
FileId: "5,ea98e40e93",
Offset: 12582912,
Size: 2097152,
ModifiedTsNs: 1634447487595829000,
FileId: "5,ea98e40e93",
Offset: 12582912,
Size: 2097152,
Mtime: 1634447487595829000,
},
{
FileId: "5,e165661172",
Offset: 14680064,
Size: 2097152,
ModifiedTsNs: 1634447487595829000,
FileId: "5,e165661172",
Offset: 14680064,
Size: 2097152,
Mtime: 1634447487595829000,
},
{
FileId: "3,e692097486",
Offset: 16777216,
Size: 2097152,
ModifiedTsNs: 1634447487595830000,
FileId: "3,e692097486",
Offset: 16777216,
Size: 2097152,
Mtime: 1634447487595830000,
},
{
FileId: "3,e28e2e3cbd",
Offset: 18874368,
Size: 2097152,
ModifiedTsNs: 1634447487595830000,
FileId: "3,e28e2e3cbd",
Offset: 18874368,
Size: 2097152,
Mtime: 1634447487595830000,
},
{
FileId: "3,e443974d4e",
Offset: 20971520,
Size: 2097152,
ModifiedTsNs: 1634447487595830000,
FileId: "3,e443974d4e",
Offset: 20971520,
Size: 2097152,
Mtime: 1634447487595830000,
},
{
FileId: "2,e815bed597",
Offset: 23068672,
Size: 2097152,
ModifiedTsNs: 1634447487595831000,
FileId: "2,e815bed597",
Offset: 23068672,
Size: 2097152,
Mtime: 1634447487595831000,
},
{
FileId: "5,e94715199e",
Offset: 25165824,
Size: 1974736,
ModifiedTsNs: 1634447487595832000,
FileId: "5,e94715199e",
Offset: 25165824,
Size: 1974736,
Mtime: 1634447487595832000,
},
}
visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
visibles := readResolvedChunks(chunks)
for x := visibles.Front(); x != nil; x = x.Next {
visible := x.Value
fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
}
}
func TestActualReadResolvedChunks2(t *testing.T) {
chunks := []*filer_pb.FileChunk{
{
FileId: "1,e7b96fef48",
Offset: 0,
Size: 184320,
ModifiedTsNs: 1,
},
{
FileId: "2,22562640b9",
Offset: 184320,
Size: 4096,
ModifiedTsNs: 2,
},
{
FileId: "2,33562640b9",
Offset: 184320,
Size: 4096,
ModifiedTsNs: 4,
},
{
FileId: "4,df033e0fe4",
Offset: 188416,
Size: 2097152,
ModifiedTsNs: 3,
},
}
visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
for x := visibles.Front(); x != nil; x = x.Next {
visible := x.Value
fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
for _, visible := range visibles {
fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTime)
}
}
+177 -193
View File
@@ -15,10 +15,10 @@ import (
func TestCompactFileChunks(t *testing.T) {
chunks := []*filer_pb.FileChunk{
{Offset: 10, Size: 100, FileId: "abc", ModifiedTsNs: 50},
{Offset: 100, Size: 100, FileId: "def", ModifiedTsNs: 100},
{Offset: 200, Size: 100, FileId: "ghi", ModifiedTsNs: 200},
{Offset: 110, Size: 200, FileId: "jkl", ModifiedTsNs: 300},
{Offset: 10, Size: 100, FileId: "abc", Mtime: 50},
{Offset: 100, Size: 100, FileId: "def", Mtime: 100},
{Offset: 200, Size: 100, FileId: "ghi", Mtime: 200},
{Offset: 110, Size: 200, FileId: "jkl", Mtime: 300},
}
compacted, garbage := CompactFileChunks(nil, chunks)
@@ -35,22 +35,22 @@ func TestCompactFileChunks(t *testing.T) {
func TestCompactFileChunks2(t *testing.T) {
chunks := []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 50},
{Offset: 100, Size: 100, FileId: "def", ModifiedTsNs: 100},
{Offset: 200, Size: 100, FileId: "ghi", ModifiedTsNs: 200},
{Offset: 0, Size: 100, FileId: "abcf", ModifiedTsNs: 300},
{Offset: 50, Size: 100, FileId: "fhfh", ModifiedTsNs: 400},
{Offset: 100, Size: 100, FileId: "yuyu", ModifiedTsNs: 500},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 50},
{Offset: 100, Size: 100, FileId: "def", Mtime: 100},
{Offset: 200, Size: 100, FileId: "ghi", Mtime: 200},
{Offset: 0, Size: 100, FileId: "abcf", Mtime: 300},
{Offset: 50, Size: 100, FileId: "fhfh", Mtime: 400},
{Offset: 100, Size: 100, FileId: "yuyu", Mtime: 500},
}
k := 3
for n := 0; n < k; n++ {
chunks = append(chunks, &filer_pb.FileChunk{
Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), ModifiedTsNs: int64(n),
Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), Mtime: int64(n),
})
chunks = append(chunks, &filer_pb.FileChunk{
Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), ModifiedTsNs: int64(n + k),
Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), Mtime: int64(n + k),
})
}
@@ -78,11 +78,11 @@ func TestRandomFileChunksCompact(t *testing.T) {
stop = start + 16
}
chunk := &filer_pb.FileChunk{
FileId: strconv.Itoa(i),
Offset: int64(start),
Size: uint64(stop - start),
ModifiedTsNs: int64(i),
Fid: &filer_pb.FileId{FileKey: uint64(i)},
FileId: strconv.Itoa(i),
Offset: int64(start),
Size: uint64(stop - start),
Mtime: int64(i),
Fid: &filer_pb.FileId{FileKey: uint64(i)},
}
chunks = append(chunks, chunk)
for x := start; x < stop; x++ {
@@ -92,8 +92,7 @@ func TestRandomFileChunksCompact(t *testing.T) {
visibles, _ := NonOverlappingVisibleIntervals(nil, chunks, 0, math.MaxInt64)
for visible := visibles.Front(); visible != nil; visible = visible.Next {
v := visible.Value
for _, v := range visibles {
for x := v.start; x < v.stop; x++ {
assert.Equal(t, strconv.Itoa(int(data[x])), v.fileId)
}
@@ -110,9 +109,9 @@ func TestIntervalMerging(t *testing.T) {
// case 0: normal
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 100, Size: 100, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 200, Size: 100, FileId: "fsad", ModifiedTsNs: 353},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 100, Size: 100, FileId: "asdf", Mtime: 134},
{Offset: 200, Size: 100, FileId: "fsad", Mtime: 353},
},
Expected: []*VisibleInterval{
{start: 0, stop: 100, fileId: "abc"},
@@ -123,8 +122,8 @@ func TestIntervalMerging(t *testing.T) {
// case 1: updates overwrite full chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
},
Expected: []*VisibleInterval{
{start: 0, stop: 200, fileId: "asdf"},
@@ -133,20 +132,20 @@ func TestIntervalMerging(t *testing.T) {
// case 2: updates overwrite part of previous chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "a", ModifiedTsNs: 123},
{Offset: 0, Size: 70, FileId: "b", ModifiedTsNs: 134},
{Offset: 0, Size: 100, FileId: "a", Mtime: 123},
{Offset: 0, Size: 70, FileId: "b", Mtime: 134},
},
Expected: []*VisibleInterval{
{start: 0, stop: 70, fileId: "b"},
{start: 70, stop: 100, fileId: "a", offsetInChunk: 70},
{start: 70, stop: 100, fileId: "a", chunkOffset: 70},
},
},
// case 3: updates overwrite full chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 50, Size: 250, FileId: "xxxx", ModifiedTsNs: 154},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
{Offset: 50, Size: 250, FileId: "xxxx", Mtime: 154},
},
Expected: []*VisibleInterval{
{start: 0, stop: 50, fileId: "asdf"},
@@ -156,9 +155,9 @@ func TestIntervalMerging(t *testing.T) {
// case 4: updates far away from prev chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 250, Size: 250, FileId: "xxxx", ModifiedTsNs: 154},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
{Offset: 250, Size: 250, FileId: "xxxx", Mtime: 154},
},
Expected: []*VisibleInterval{
{start: 0, stop: 200, fileId: "asdf"},
@@ -168,22 +167,22 @@ func TestIntervalMerging(t *testing.T) {
// case 5: updates overwrite full chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "a", ModifiedTsNs: 123},
{Offset: 0, Size: 200, FileId: "d", ModifiedTsNs: 184},
{Offset: 70, Size: 150, FileId: "c", ModifiedTsNs: 143},
{Offset: 80, Size: 100, FileId: "b", ModifiedTsNs: 134},
{Offset: 0, Size: 100, FileId: "a", Mtime: 123},
{Offset: 0, Size: 200, FileId: "d", Mtime: 184},
{Offset: 70, Size: 150, FileId: "c", Mtime: 143},
{Offset: 80, Size: 100, FileId: "b", Mtime: 134},
},
Expected: []*VisibleInterval{
{start: 0, stop: 200, fileId: "d"},
{start: 200, stop: 220, fileId: "c", offsetInChunk: 130},
{start: 200, stop: 220, fileId: "c", chunkOffset: 130},
},
},
// case 6: same updates
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", Fid: &filer_pb.FileId{FileKey: 1}, ModifiedTsNs: 123},
{Offset: 0, Size: 100, FileId: "axf", Fid: &filer_pb.FileId{FileKey: 2}, ModifiedTsNs: 124},
{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, ModifiedTsNs: 125},
{Offset: 0, Size: 100, FileId: "abc", Fid: &filer_pb.FileId{FileKey: 1}, Mtime: 123},
{Offset: 0, Size: 100, FileId: "axf", Fid: &filer_pb.FileId{FileKey: 2}, Mtime: 123},
{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, Mtime: 123},
},
Expected: []*VisibleInterval{
{start: 0, stop: 100, fileId: "xyz"},
@@ -192,12 +191,12 @@ func TestIntervalMerging(t *testing.T) {
// case 7: real updates
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 2097152, FileId: "7,0294cbb9892b", ModifiedTsNs: 123},
{Offset: 0, Size: 3145728, FileId: "3,029565bf3092", ModifiedTsNs: 130},
{Offset: 2097152, Size: 3145728, FileId: "6,029632f47ae2", ModifiedTsNs: 140},
{Offset: 5242880, Size: 3145728, FileId: "2,029734c5aa10", ModifiedTsNs: 150},
{Offset: 8388608, Size: 3145728, FileId: "5,02982f80de50", ModifiedTsNs: 160},
{Offset: 11534336, Size: 2842193, FileId: "7,0299ad723803", ModifiedTsNs: 170},
{Offset: 0, Size: 2097152, FileId: "7,0294cbb9892b", Mtime: 123},
{Offset: 0, Size: 3145728, FileId: "3,029565bf3092", Mtime: 130},
{Offset: 2097152, Size: 3145728, FileId: "6,029632f47ae2", Mtime: 140},
{Offset: 5242880, Size: 3145728, FileId: "2,029734c5aa10", Mtime: 150},
{Offset: 8388608, Size: 3145728, FileId: "5,02982f80de50", Mtime: 160},
{Offset: 11534336, Size: 2842193, FileId: "7,0299ad723803", Mtime: 170},
},
Expected: []*VisibleInterval{
{start: 0, stop: 2097152, fileId: "3,029565bf3092"},
@@ -210,11 +209,11 @@ func TestIntervalMerging(t *testing.T) {
// case 8: real bug
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 77824, FileId: "4,0b3df938e301", ModifiedTsNs: 123},
{Offset: 471040, Size: 472225 - 471040, FileId: "6,0b3e0650019c", ModifiedTsNs: 130},
{Offset: 77824, Size: 208896 - 77824, FileId: "4,0b3f0c7202f0", ModifiedTsNs: 140},
{Offset: 208896, Size: 339968 - 208896, FileId: "2,0b4031a72689", ModifiedTsNs: 150},
{Offset: 339968, Size: 471040 - 339968, FileId: "3,0b416a557362", ModifiedTsNs: 160},
{Offset: 0, Size: 77824, FileId: "4,0b3df938e301", Mtime: 123},
{Offset: 471040, Size: 472225 - 471040, FileId: "6,0b3e0650019c", Mtime: 130},
{Offset: 77824, Size: 208896 - 77824, FileId: "4,0b3f0c7202f0", Mtime: 140},
{Offset: 208896, Size: 339968 - 208896, FileId: "2,0b4031a72689", Mtime: 150},
{Offset: 339968, Size: 471040 - 339968, FileId: "3,0b416a557362", Mtime: 160},
},
Expected: []*VisibleInterval{
{start: 0, stop: 77824, fileId: "4,0b3df938e301"},
@@ -229,17 +228,11 @@ func TestIntervalMerging(t *testing.T) {
for i, testcase := range testcases {
log.Printf("++++++++++ merged test case %d ++++++++++++++++++++", i)
intervals, _ := NonOverlappingVisibleIntervals(nil, testcase.Chunks, 0, math.MaxInt64)
x := -1
for visible := intervals.Front(); visible != nil; visible = visible.Next {
x++
interval := visible.Value
log.Printf("test case %d, interval start=%d, stop=%d, fileId=%s",
i, interval.start, interval.stop, interval.fileId)
for x, interval := range intervals {
log.Printf("test case %d, interval %d, start=%d, stop=%d, fileId=%s",
i, x, interval.start, interval.stop, interval.fileId)
}
x = -1
for visible := intervals.Front(); visible != nil; visible = visible.Next {
x++
interval := visible.Value
for x, interval := range intervals {
if interval.start != testcase.Expected[x].start {
t.Fatalf("failed on test case %d, interval %d, start %d, expect %d",
i, x, interval.start, testcase.Expected[x].start)
@@ -252,13 +245,13 @@ func TestIntervalMerging(t *testing.T) {
t.Fatalf("failed on test case %d, interval %d, chunkId %s, expect %s",
i, x, interval.fileId, testcase.Expected[x].fileId)
}
if interval.offsetInChunk != testcase.Expected[x].offsetInChunk {
t.Fatalf("failed on test case %d, interval %d, offsetInChunk %d, expect %d",
i, x, interval.offsetInChunk, testcase.Expected[x].offsetInChunk)
if interval.chunkOffset != testcase.Expected[x].chunkOffset {
t.Fatalf("failed on test case %d, interval %d, chunkOffset %d, expect %d",
i, x, interval.chunkOffset, testcase.Expected[x].chunkOffset)
}
}
if intervals.Len() != len(testcase.Expected) {
t.Fatalf("failed to compact test case %d, len %d expected %d", i, intervals.Len(), len(testcase.Expected))
if len(intervals) != len(testcase.Expected) {
t.Fatalf("failed to compact test case %d, len %d expected %d", i, len(intervals), len(testcase.Expected))
}
}
@@ -276,147 +269,147 @@ func TestChunksReading(t *testing.T) {
// case 0: normal
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 100, Size: 100, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 200, Size: 100, FileId: "fsad", ModifiedTsNs: 353},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 100, Size: 100, FileId: "asdf", Mtime: 134},
{Offset: 200, Size: 100, FileId: "fsad", Mtime: 353},
},
Offset: 0,
Size: 250,
Expected: []*ChunkView{
{OffsetInChunk: 0, ViewSize: 100, FileId: "abc", ViewOffset: 0},
{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 100},
{OffsetInChunk: 0, ViewSize: 50, FileId: "fsad", ViewOffset: 200},
{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
{Offset: 0, Size: 50, FileId: "fsad", LogicOffset: 200},
},
},
// case 1: updates overwrite full chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
},
Offset: 50,
Size: 100,
Expected: []*ChunkView{
{OffsetInChunk: 50, ViewSize: 100, FileId: "asdf", ViewOffset: 50},
{Offset: 50, Size: 100, FileId: "asdf", LogicOffset: 50},
},
},
// case 2: updates overwrite part of previous chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 3, Size: 100, FileId: "a", ModifiedTsNs: 123},
{Offset: 10, Size: 50, FileId: "b", ModifiedTsNs: 134},
{Offset: 3, Size: 100, FileId: "a", Mtime: 123},
{Offset: 10, Size: 50, FileId: "b", Mtime: 134},
},
Offset: 30,
Size: 40,
Expected: []*ChunkView{
{OffsetInChunk: 20, ViewSize: 30, FileId: "b", ViewOffset: 30},
{OffsetInChunk: 57, ViewSize: 10, FileId: "a", ViewOffset: 60},
{Offset: 20, Size: 30, FileId: "b", LogicOffset: 30},
{Offset: 57, Size: 10, FileId: "a", LogicOffset: 60},
},
},
// case 3: updates overwrite full chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 50, Size: 250, FileId: "xxxx", ModifiedTsNs: 154},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
{Offset: 50, Size: 250, FileId: "xxxx", Mtime: 154},
},
Offset: 0,
Size: 200,
Expected: []*ChunkView{
{OffsetInChunk: 0, ViewSize: 50, FileId: "asdf", ViewOffset: 0},
{OffsetInChunk: 0, ViewSize: 150, FileId: "xxxx", ViewOffset: 50},
{Offset: 0, Size: 50, FileId: "asdf", LogicOffset: 0},
{Offset: 0, Size: 150, FileId: "xxxx", LogicOffset: 50},
},
},
// case 4: updates far away from prev chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 250, Size: 250, FileId: "xxxx", ModifiedTsNs: 154},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
{Offset: 250, Size: 250, FileId: "xxxx", Mtime: 154},
},
Offset: 0,
Size: 400,
Expected: []*ChunkView{
{OffsetInChunk: 0, ViewSize: 200, FileId: "asdf", ViewOffset: 0},
{OffsetInChunk: 0, ViewSize: 150, FileId: "xxxx", ViewOffset: 250},
{Offset: 0, Size: 200, FileId: "asdf", LogicOffset: 0},
{Offset: 0, Size: 150, FileId: "xxxx", LogicOffset: 250},
},
},
// case 5: updates overwrite full chunks
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "a", ModifiedTsNs: 123},
{Offset: 0, Size: 200, FileId: "c", ModifiedTsNs: 184},
{Offset: 70, Size: 150, FileId: "b", ModifiedTsNs: 143},
{Offset: 80, Size: 100, FileId: "xxxx", ModifiedTsNs: 134},
{Offset: 0, Size: 100, FileId: "a", Mtime: 123},
{Offset: 0, Size: 200, FileId: "c", Mtime: 184},
{Offset: 70, Size: 150, FileId: "b", Mtime: 143},
{Offset: 80, Size: 100, FileId: "xxxx", Mtime: 134},
},
Offset: 0,
Size: 220,
Expected: []*ChunkView{
{OffsetInChunk: 0, ViewSize: 200, FileId: "c", ViewOffset: 0},
{OffsetInChunk: 130, ViewSize: 20, FileId: "b", ViewOffset: 200},
{Offset: 0, Size: 200, FileId: "c", LogicOffset: 0},
{Offset: 130, Size: 20, FileId: "b", LogicOffset: 200},
},
},
// case 6: same updates
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", Fid: &filer_pb.FileId{FileKey: 1}, ModifiedTsNs: 123},
{Offset: 0, Size: 100, FileId: "def", Fid: &filer_pb.FileId{FileKey: 2}, ModifiedTsNs: 124},
{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, ModifiedTsNs: 125},
{Offset: 0, Size: 100, FileId: "abc", Fid: &filer_pb.FileId{FileKey: 1}, Mtime: 123},
{Offset: 0, Size: 100, FileId: "def", Fid: &filer_pb.FileId{FileKey: 2}, Mtime: 123},
{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, Mtime: 123},
},
Offset: 0,
Size: 100,
Expected: []*ChunkView{
{OffsetInChunk: 0, ViewSize: 100, FileId: "xyz", ViewOffset: 0},
{Offset: 0, Size: 100, FileId: "xyz", LogicOffset: 0},
},
},
// case 7: edge cases
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 100, Size: 100, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 200, Size: 100, FileId: "fsad", ModifiedTsNs: 353},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 100, Size: 100, FileId: "asdf", Mtime: 134},
{Offset: 200, Size: 100, FileId: "fsad", Mtime: 353},
},
Offset: 0,
Size: 200,
Expected: []*ChunkView{
{OffsetInChunk: 0, ViewSize: 100, FileId: "abc", ViewOffset: 0},
{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 100},
{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
},
},
// case 8: edge cases
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
{Offset: 90, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
{Offset: 190, Size: 300, FileId: "fsad", ModifiedTsNs: 353},
{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
{Offset: 90, Size: 200, FileId: "asdf", Mtime: 134},
{Offset: 190, Size: 300, FileId: "fsad", Mtime: 353},
},
Offset: 0,
Size: 300,
Expected: []*ChunkView{
{OffsetInChunk: 0, ViewSize: 90, FileId: "abc", ViewOffset: 0},
{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 90},
{OffsetInChunk: 0, ViewSize: 110, FileId: "fsad", ViewOffset: 190},
{Offset: 0, Size: 90, FileId: "abc", LogicOffset: 0},
{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 90},
{Offset: 0, Size: 110, FileId: "fsad", LogicOffset: 190},
},
},
// case 9: edge cases
{
Chunks: []*filer_pb.FileChunk{
{Offset: 0, Size: 43175947, FileId: "2,111fc2cbfac1", ModifiedTsNs: 1},
{Offset: 43175936, Size: 52981771 - 43175936, FileId: "2,112a36ea7f85", ModifiedTsNs: 2},
{Offset: 52981760, Size: 72564747 - 52981760, FileId: "4,112d5f31c5e7", ModifiedTsNs: 3},
{Offset: 72564736, Size: 133255179 - 72564736, FileId: "1,113245f0cdb6", ModifiedTsNs: 4},
{Offset: 133255168, Size: 137269259 - 133255168, FileId: "3,1141a70733b5", ModifiedTsNs: 5},
{Offset: 137269248, Size: 153578836 - 137269248, FileId: "1,114201d5bbdb", ModifiedTsNs: 6},
{Offset: 0, Size: 43175947, FileId: "2,111fc2cbfac1", Mtime: 1},
{Offset: 43175936, Size: 52981771 - 43175936, FileId: "2,112a36ea7f85", Mtime: 2},
{Offset: 52981760, Size: 72564747 - 52981760, FileId: "4,112d5f31c5e7", Mtime: 3},
{Offset: 72564736, Size: 133255179 - 72564736, FileId: "1,113245f0cdb6", Mtime: 4},
{Offset: 133255168, Size: 137269259 - 133255168, FileId: "3,1141a70733b5", Mtime: 5},
{Offset: 137269248, Size: 153578836 - 137269248, FileId: "1,114201d5bbdb", Mtime: 6},
},
Offset: 0,
Size: 153578836,
Expected: []*ChunkView{
{OffsetInChunk: 0, ViewSize: 43175936, FileId: "2,111fc2cbfac1", ViewOffset: 0},
{OffsetInChunk: 0, ViewSize: 52981760 - 43175936, FileId: "2,112a36ea7f85", ViewOffset: 43175936},
{OffsetInChunk: 0, ViewSize: 72564736 - 52981760, FileId: "4,112d5f31c5e7", ViewOffset: 52981760},
{OffsetInChunk: 0, ViewSize: 133255168 - 72564736, FileId: "1,113245f0cdb6", ViewOffset: 72564736},
{OffsetInChunk: 0, ViewSize: 137269248 - 133255168, FileId: "3,1141a70733b5", ViewOffset: 133255168},
{OffsetInChunk: 0, ViewSize: 153578836 - 137269248, FileId: "1,114201d5bbdb", ViewOffset: 137269248},
{Offset: 0, Size: 43175936, FileId: "2,111fc2cbfac1", LogicOffset: 0},
{Offset: 0, Size: 52981760 - 43175936, FileId: "2,112a36ea7f85", LogicOffset: 43175936},
{Offset: 0, Size: 72564736 - 52981760, FileId: "4,112d5f31c5e7", LogicOffset: 52981760},
{Offset: 0, Size: 133255168 - 72564736, FileId: "1,113245f0cdb6", LogicOffset: 72564736},
{Offset: 0, Size: 137269248 - 133255168, FileId: "3,1141a70733b5", LogicOffset: 133255168},
{Offset: 0, Size: 153578836 - 137269248, FileId: "1,114201d5bbdb", LogicOffset: 137269248},
},
},
}
@@ -427,31 +420,28 @@ func TestChunksReading(t *testing.T) {
}
log.Printf("++++++++++ read test case %d ++++++++++++++++++++", i)
chunks := ViewFromChunks(nil, testcase.Chunks, testcase.Offset, testcase.Size)
x := -1
for c := chunks.Front(); c != nil; c = c.Next {
x++
chunk := c.Value
for x, chunk := range chunks {
log.Printf("read case %d, chunk %d, offset=%d, size=%d, fileId=%s",
i, x, chunk.OffsetInChunk, chunk.ViewSize, chunk.FileId)
if chunk.OffsetInChunk != testcase.Expected[x].OffsetInChunk {
i, x, chunk.Offset, chunk.Size, chunk.FileId)
if chunk.Offset != testcase.Expected[x].Offset {
t.Fatalf("failed on read case %d, chunk %s, Offset %d, expect %d",
i, chunk.FileId, chunk.OffsetInChunk, testcase.Expected[x].OffsetInChunk)
i, chunk.FileId, chunk.Offset, testcase.Expected[x].Offset)
}
if chunk.ViewSize != testcase.Expected[x].ViewSize {
t.Fatalf("failed on read case %d, chunk %s, ViewSize %d, expect %d",
i, chunk.FileId, chunk.ViewSize, testcase.Expected[x].ViewSize)
if chunk.Size != testcase.Expected[x].Size {
t.Fatalf("failed on read case %d, chunk %s, Size %d, expect %d",
i, chunk.FileId, chunk.Size, testcase.Expected[x].Size)
}
if chunk.FileId != testcase.Expected[x].FileId {
t.Fatalf("failed on read case %d, chunk %d, FileId %s, expect %s",
i, x, chunk.FileId, testcase.Expected[x].FileId)
}
if chunk.ViewOffset != testcase.Expected[x].ViewOffset {
t.Fatalf("failed on read case %d, chunk %d, ViewOffset %d, expect %d",
i, x, chunk.ViewOffset, testcase.Expected[x].ViewOffset)
if chunk.LogicOffset != testcase.Expected[x].LogicOffset {
t.Fatalf("failed on read case %d, chunk %d, LogicOffset %d, expect %d",
i, x, chunk.LogicOffset, testcase.Expected[x].LogicOffset)
}
}
if chunks.Len() != len(testcase.Expected) {
t.Fatalf("failed to read test case %d, len %d expected %d", i, chunks.Len(), len(testcase.Expected))
if len(chunks) != len(testcase.Expected) {
t.Fatalf("failed to read test case %d, len %d expected %d", i, len(chunks), len(testcase.Expected))
}
}
@@ -465,10 +455,10 @@ func BenchmarkCompactFileChunks(b *testing.B) {
for n := 0; n < k; n++ {
chunks = append(chunks, &filer_pb.FileChunk{
Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), ModifiedTsNs: int64(n),
Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), Mtime: int64(n),
})
chunks = append(chunks, &filer_pb.FileChunk{
Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), ModifiedTsNs: int64(n + k),
Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), Mtime: int64(n + k),
})
}
@@ -477,79 +467,73 @@ func BenchmarkCompactFileChunks(b *testing.B) {
}
}
func addVisibleInterval(visibles *IntervalList[*VisibleInterval], x *VisibleInterval) {
visibles.AppendInterval(&Interval[*VisibleInterval]{
StartOffset: x.start,
StopOffset: x.stop,
TsNs: x.modifiedTsNs,
Value: x,
})
}
func TestViewFromVisibleIntervals(t *testing.T) {
visibles := NewIntervalList[*VisibleInterval]()
addVisibleInterval(visibles, &VisibleInterval{
start: 0,
stop: 25,
fileId: "fid1",
})
addVisibleInterval(visibles, &VisibleInterval{
start: 4096,
stop: 8192,
fileId: "fid2",
})
addVisibleInterval(visibles, &VisibleInterval{
start: 16384,
stop: 18551,
fileId: "fid3",
})
visibles := []VisibleInterval{
{
start: 0,
stop: 25,
fileId: "fid1",
},
{
start: 4096,
stop: 8192,
fileId: "fid2",
},
{
start: 16384,
stop: 18551,
fileId: "fid3",
},
}
views := ViewFromVisibleIntervals(visibles, 0, math.MaxInt32)
if views.Len() != visibles.Len() {
assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
if len(views) != len(visibles) {
assert.Equal(t, len(visibles), len(views), "ViewFromVisibleIntervals error")
}
}
func TestViewFromVisibleIntervals2(t *testing.T) {
visibles := NewIntervalList[*VisibleInterval]()
addVisibleInterval(visibles, &VisibleInterval{
start: 344064,
stop: 348160,
fileId: "fid1",
})
addVisibleInterval(visibles, &VisibleInterval{
start: 348160,
stop: 356352,
fileId: "fid2",
})
visibles := []VisibleInterval{
{
start: 344064,
stop: 348160,
fileId: "fid1",
},
{
start: 348160,
stop: 356352,
fileId: "fid2",
},
}
views := ViewFromVisibleIntervals(visibles, 0, math.MaxInt32)
if views.Len() != visibles.Len() {
assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
if len(views) != len(visibles) {
assert.Equal(t, len(visibles), len(views), "ViewFromVisibleIntervals error")
}
}
func TestViewFromVisibleIntervals3(t *testing.T) {
visibles := NewIntervalList[*VisibleInterval]()
addVisibleInterval(visibles, &VisibleInterval{
start: 1000,
stop: 2000,
fileId: "fid1",
})
addVisibleInterval(visibles, &VisibleInterval{
start: 3000,
stop: 4000,
fileId: "fid2",
})
visibles := []VisibleInterval{
{
start: 1000,
stop: 2000,
fileId: "fid1",
},
{
start: 3000,
stop: 4000,
fileId: "fid2",
},
}
views := ViewFromVisibleIntervals(visibles, 1700, 1500)
if views.Len() != visibles.Len() {
assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
if len(views) != len(visibles) {
assert.Equal(t, len(visibles), len(views), "ViewFromVisibleIntervals error")
}
}

Some files were not shown because too many files have changed in this diff Show More