mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-03 21:21:57 +00:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 chrislu
|
2f86f10fe1 | ||
|
chrislu
|
264c138c1d | ||
|
chrislu
|
24317a9763 | ||
|
chrislu
|
3da8d7309a | ||
|
chrislu
|
648b33beb0 | ||
|
chrislu
|
1484cb224f | ||
|
chrislu
|
600d2f92a4 | ||
|
chrislu
|
15b6c7a6f0 | ||
|
chrislu
|
bfd0601fd3 | ||
|
chrislu
|
977c8e7830 | ||
|
chrislu
|
e264fa6e28 | ||
|
chrislu
|
dcbded20bb | ||
|
chrislu
|
1c79301f89 | ||
|
chrislu
|
956ce6416f | ||
|
chrislu
|
3f4400dc03 | ||
|
chrislu
|
daa086fd72 | ||
|
chrislu
|
7e33915a26 |
@@ -38,13 +38,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -87,13 +87,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -16,11 +16,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -33,30 +33,30 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -17,11 +17,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -34,30 +34,30 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -16,11 +16,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -34,20 +34,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -17,11 +17,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -35,20 +35,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -17,11 +17,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -35,20 +35,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -16,11 +16,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -34,20 +34,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -16,11 +16,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -34,20 +34,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@c090f4e553673e6e505ea70d6a95362ee12adb94
|
||||
uses: actions/dependency-review-action@23d1ffffb6fa5401173051ec21eba8c35242733f
|
||||
|
||||
+15
-30
@@ -24,13 +24,13 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
|
||||
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
@@ -42,43 +42,28 @@ jobs:
|
||||
run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Run FIO
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
echo "Starting FIO at: $(date)"
|
||||
# Concurrent r/w
|
||||
echo 'Run randrw with size=16M bs=4k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
|
||||
echo "Verify FIO at: $(date)"
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
|
||||
echo 'Run randrw with size=16M bs=128k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
|
||||
echo 'Run randrw with size=16M bs=1m'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
|
||||
# Verified write
|
||||
echo 'Run randwrite with size=16M bs=4k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
|
||||
- name: Run FIO
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
echo "Starting FIO at: $(date)"
|
||||
# Concurrent r/w
|
||||
echo 'Run randrw with size=16M bs=128k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
|
||||
echo "Verify FIO at: $(date)"
|
||||
# Verified write
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
|
||||
echo 'Run randwrite with size=16M bs=128k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
|
||||
- name: Run FIO
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
echo "Starting FIO at: $(date)"
|
||||
# Concurrent r/w
|
||||
echo 'Run randrw with size=16M bs=1m'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
|
||||
echo "Verify FIO at: $(date)"
|
||||
# Verified write
|
||||
echo 'Run randwrite with size=16M bs=1m'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
|
||||
- name: Save logs
|
||||
if: always()
|
||||
|
||||
@@ -21,13 +21,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
|
||||
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as
|
||||
contributors and maintainers pledge to make participation in our project and
|
||||
our community a harassment-free experience for everyone, regardless of age, body
|
||||
size, disability, ethnicity, gender identity and expression, level of experience,
|
||||
nationality, personal appearance, race, religion, or sexual identity and
|
||||
orientation.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to creating a positive environment
|
||||
include:
|
||||
|
||||
- Using welcoming and inclusive language
|
||||
- Being respectful of differing viewpoints and experiences
|
||||
- Gracefully accepting constructive criticism
|
||||
- Focusing on what is best for the community
|
||||
- Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
- The use of sexualized language or imagery and unwelcome sexual attention or
|
||||
advances
|
||||
- Trolling, insulting/derogatory comments, and personal or political attacks
|
||||
- Public or private harassment
|
||||
- Publishing others' private information, such as a physical or electronic
|
||||
address, without explicit permission
|
||||
- Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Our Responsibilities
|
||||
|
||||
Project maintainers are responsible for clarifying the standards of acceptable
|
||||
behavior and are expected to take appropriate and fair corrective action in
|
||||
response to any instances of unacceptable behavior.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or
|
||||
reject comments, commits, code, wiki edits, issues, and other contributions
|
||||
that are not aligned to this Code of Conduct, or to ban temporarily or
|
||||
permanently any contributor for other behaviors that they deem inappropriate,
|
||||
threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies both within project spaces and in public spaces
|
||||
when an individual is representing the project or its community. Examples of
|
||||
representing a project or community include using an official project e-mail
|
||||
address, posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event. Representation of a project may be
|
||||
further defined and clarified by project maintainers.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported by contacting the project team at <enteremailhere>. All
|
||||
complaints will be reviewed and investigated and will result in a response that
|
||||
is deemed necessary and appropriate to the circumstances. The project team is
|
||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
||||
Further details of specific enforcement policies may be posted separately.
|
||||
|
||||
Project maintainers who do not follow or enforce the Code of Conduct in good
|
||||
faith may face temporary or permanent repercussions as determined by other
|
||||
members of the project's leadership.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
||||
available at [http://contributor-covenant.org/version/1/4][version]
|
||||
|
||||
[homepage]: http://contributor-covenant.org
|
||||
[version]: http://contributor-covenant.org/version/1/4/
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
[](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
|
||||
[](https://twitter.com/intent/follow?screen_name=seaweedfs)
|
||||
[](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
|
||||
[](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
|
||||
[](https://godoc.org/github.com/seaweedfs/seaweedfs/weed)
|
||||
[](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
[](https://hub.docker.com/r/chrislusf/seaweedfs/)
|
||||
@@ -79,7 +79,7 @@ Table of Contents
|
||||
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
|
||||
|
||||
## Quick Start with Single Binary ##
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
|
||||
- [Evercam Camera Management Software](https://evercam.io/)
|
||||
- [Spherical Elephant GmbH](https://www.sphericalelephant.com)
|
||||
- [WizardTales GmbH](https://www.wizardtales.com)
|
||||
|
||||
- <h2 align="center">Backers</h2>
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.20-alpine as builder
|
||||
FROM golang:1.19-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.20-buster as builder
|
||||
FROM golang:1.19-buster as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.20-buster as builder
|
||||
FROM golang:1.19-buster as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
+18
-19
@@ -39,64 +39,63 @@ s3tests_build:
|
||||
docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
|
||||
|
||||
dev: build
|
||||
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
dev_race: binary_race
|
||||
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
dev_tls: build certstrap
|
||||
ENV_FILE="tls.env" docker compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
dev_mount: build
|
||||
docker compose -f compose/local-mount-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
|
||||
|
||||
run_image: build
|
||||
docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local
|
||||
|
||||
profile_mount: build
|
||||
docker compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
|
||||
|
||||
k8s: build
|
||||
docker compose -f compose/local-k8s-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
|
||||
|
||||
dev_registry: build
|
||||
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-registry-compose.yml -p seaweedfs up
|
||||
|
||||
dev_replicate:
|
||||
docker build --build-arg TAGS=gocdk --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.go_build .
|
||||
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
|
||||
dev_replicate: build
|
||||
docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
|
||||
|
||||
dev_auditlog: build
|
||||
docker compose -f compose/local-auditlog-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
|
||||
|
||||
dev_nextcloud: build
|
||||
docker compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
|
||||
|
||||
cluster: build
|
||||
docker compose -f compose/local-cluster-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
|
||||
|
||||
2clusters: build
|
||||
docker compose -f compose/local-clusters-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
|
||||
|
||||
2mount: build
|
||||
docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
|
||||
|
||||
hashicorp_raft: build
|
||||
docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
|
||||
|
||||
s3tests: build s3tests_build
|
||||
docker compose -f compose/local-s3tests-compose.yml -p seaweedfs up
|
||||
docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
|
||||
|
||||
filer_etcd: build
|
||||
docker stack deploy -c compose/swarm-etcd.yml fs
|
||||
|
||||
test_etcd: build
|
||||
docker compose -f compose/test-etcd-filer.yml -p seaweedfs up
|
||||
docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
|
||||
|
||||
test_ydb: tags = ydb
|
||||
test_ydb: build
|
||||
export
|
||||
docker compose -f compose/test-ydb-filer.yml -p seaweedfs up
|
||||
docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up
|
||||
|
||||
clean:
|
||||
rm ./weed
|
||||
|
||||
+2
-12
@@ -1,15 +1,5 @@
|
||||
# Docker
|
||||
|
||||
## Compose V2
|
||||
SeaweedFS now uses the `v2` syntax `docker compose`
|
||||
|
||||
If you rely on using Docker Compose as docker-compose (with a hyphen), you can set up Compose V2 to act as a drop-in replacement of the previous docker-compose. Refer to the [Installing Compose](https://docs.docker.com/compose/install/) section for detailed instructions on upgrading.
|
||||
|
||||
Confirm your system has docker compose v2 with a version check
|
||||
```bash
|
||||
$ docker compose version
|
||||
Docker Compose version v2.10.2
|
||||
```
|
||||
|
||||
## Try it out
|
||||
|
||||
@@ -17,7 +7,7 @@ Docker Compose version v2.10.2
|
||||
|
||||
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml
|
||||
|
||||
docker compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
docker-compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
|
||||
```
|
||||
|
||||
@@ -27,7 +17,7 @@ docker compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
|
||||
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml
|
||||
|
||||
docker compose -f seaweedfs-dev-compose.yml -p seaweedfs up
|
||||
docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
|
||||
|
||||
```
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
s3:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master0:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
server1:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master0:
|
||||
@@ -6,7 +6,7 @@ services:
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/0:/data
|
||||
environment:
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
ports:
|
||||
- 9334:9334
|
||||
- 19334:19334
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/1:/data
|
||||
environment:
|
||||
@@ -30,7 +30,7 @@ services:
|
||||
ports:
|
||||
- 9335:9335
|
||||
- 19335:19335
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/2:/data
|
||||
environment:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
etcd:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
ydb:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '3.9'
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -3,10 +3,10 @@ CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret';
|
||||
GRANT ALL PRIVILEGES ON seaweedfs.* TO 'seaweedfs'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
USE seaweedfs;
|
||||
CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
CREATE TABLE IF NOT EXISTS filemeta (
|
||||
dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
name VARCHAR(1000) COMMENT 'directory or file name',
|
||||
directory TEXT COMMENT 'full path to parent directory',
|
||||
meta LONGBLOB,
|
||||
PRIMARY KEY (dirhash, name)
|
||||
) DEFAULT CHARSET=utf8;
|
||||
@@ -1,52 +1,52 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.20
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.107.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.28.0
|
||||
cloud.google.com/go/storage v1.29.0
|
||||
cloud.google.com/go v0.104.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.25.1
|
||||
cloud.google.com/go/storage v1.26.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.44.189
|
||||
github.com/Shopify/sarama v1.36.0
|
||||
github.com/aws/aws-sdk-go v1.44.100
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/eapache/go-resiliency v1.3.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
|
||||
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.20.0
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.19.1
|
||||
github.com/fsnotify/fsnotify v1.5.4 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-redsync/redsync/v4 v4.7.1
|
||||
github.com/go-sql-driver/mysql v1.7.0
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.6.0
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/go-zookeeper/zk v1.0.2 // indirect
|
||||
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.1.2
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/google/go-cmp v0.5.8 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
@@ -57,42 +57,43 @@ require (
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.14 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.6
|
||||
github.com/klauspost/compress v1.15.9 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.7
|
||||
github.com/linxGnu/grocksdb v1.7.14
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/linxGnu/grocksdb v1.7.7
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.9 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/olivere/elastic/v7 v7.0.32
|
||||
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||
github.com/peterh/liner v1.2.2
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.1.0
|
||||
github.com/prometheus/client_golang v1.14.0
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/client_golang v1.13.0
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.9.0
|
||||
github.com/prometheus/procfs v0.8.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible
|
||||
github.com/seaweedfs/raft v1.1.0
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spf13/afero v1.9.3 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/spf13/afero v1.9.2 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.15.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/spf13/viper v1.13.0
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.14.4
|
||||
github.com/tidwall/gjson v1.14.3
|
||||
github.com/tidwall/match v1.1.1
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
|
||||
@@ -102,89 +103,87 @@ require (
|
||||
github.com/viant/ptrie v0.3.0
|
||||
github.com/viant/toolbox v0.33.2 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/xdg-go/scram v1.1.1 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.3 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.7
|
||||
go.mongodb.org/mongo-driver v1.11.1
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.28.0
|
||||
gocloud.dev/pubsub/natspubsub v0.28.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.27.0
|
||||
golang.org/x/crypto v0.3.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
|
||||
go.etcd.io/etcd/client/v3 v3.5.5
|
||||
go.mongodb.org/mongo-driver v1.10.2
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
gocloud.dev v0.26.0
|
||||
gocloud.dev/pubsub/natspubsub v0.26.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.26.0
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
|
||||
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.5.0
|
||||
golang.org/x/oauth2 v0.2.0 // indirect
|
||||
golang.org/x/sys v0.4.0
|
||||
golang.org/x/text v0.6.0 // indirect
|
||||
golang.org/x/tools v0.5.0
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/api v0.108.0
|
||||
golang.org/x/net v0.0.0-20220909164309-bea034e7d591
|
||||
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
|
||||
golang.org/x/sys v0.0.0-20220818161305-2296e01440c6
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.12
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
|
||||
google.golang.org/api v0.96.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
|
||||
google.golang.org/grpc v1.52.0
|
||||
google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
|
||||
google.golang.org/grpc v1.49.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/cc/v3 v3.40.0 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||
modernc.org/libc v1.22.2 // indirect
|
||||
modernc.org/cc/v3 v3.38.1 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.9 // indirect
|
||||
modernc.org/libc v1.19.0 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/memory v1.4.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/sqlite v1.20.1
|
||||
modernc.org/sqlite v1.19.1
|
||||
modernc.org/strutil v1.1.3
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Jille/raft-grpc-transport v1.4.0
|
||||
github.com/arangodb/go-driver v1.4.1
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/Jille/raft-grpc-transport v1.2.0
|
||||
github.com/arangodb/go-driver v1.3.3
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/google/flatbuffers v2.0.8+incompatible
|
||||
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
|
||||
github.com/hashicorp/raft v1.3.11
|
||||
github.com/hashicorp/raft v1.3.10
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
||||
github.com/rabbitmq/amqp091-go v1.5.0
|
||||
github.com/schollz/progressbar/v3 v3.13.0
|
||||
github.com/tikv/client-go/v2 v2.0.5
|
||||
github.com/tikv/client-go/v2 v2.0.1
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.6
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.38.0
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.14.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v0.8.0 // indirect
|
||||
cloud.google.com/go/compute v1.7.0 // indirect
|
||||
cloud.google.com/go/iam v0.3.0 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
|
||||
github.com/aws/smithy-go v1.13.4 // indirect
|
||||
github.com/armon/go-metrics v0.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
|
||||
github.com/aws/smithy-go v1.11.2 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/fclairamb/go-log v0.4.1 // indirect
|
||||
github.com/fclairamb/go-log v0.3.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.2.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
@@ -195,50 +194,43 @@ require (
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.7 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/montanaflynn/stats v0.6.6 // indirect
|
||||
github.com/nats-io/nats.go v1.20.0 // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
|
||||
github.com/nats-io/nkeys v0.3.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 // indirect
|
||||
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
|
||||
github.com/rivo/uniseg v0.4.3 // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect
|
||||
github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stathat/consistent v1.0.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f // indirect
|
||||
github.com/subosito/gotenv v1.4.1 // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220916101045-1c4d1accabb6 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.7 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/mod v0.7.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/term v0.4.0 // indirect
|
||||
go.uber.org/zap v1.21.0 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/uint128 v1.2.0 // indirect
|
||||
lukechampine.com/uint128 v1.1.1 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.42"
|
||||
version: "3.42"
|
||||
appVersion: "3.29"
|
||||
version: "3.29"
|
||||
|
||||
@@ -14,13 +14,13 @@ with ENV.
|
||||
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
CREATE TABLE IF NOT EXISTS filemeta (
|
||||
dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
name VARCHAR(1000) BINARY COMMENT 'directory or file name',
|
||||
directory TEXT BINARY COMMENT 'full path to parent directory',
|
||||
meta LONGBLOB,
|
||||
PRIMARY KEY (dirhash, name)
|
||||
) DEFAULT CHARSET=utf8;
|
||||
```
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
@@ -101,12 +101,7 @@ spec:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
exec /usr/bin/weed \
|
||||
{{- if eq .Values.filer.logs.type "hostPath" }}
|
||||
-logdir=/logs \
|
||||
{{- else }}
|
||||
-logtostderr=true \
|
||||
{{- end }}
|
||||
exec /usr/bin/weed -logdir=/logs \
|
||||
{{- if .Values.filer.loggingOverrideLevel }}
|
||||
-v={{ .Values.filer.loggingOverrideLevel }} \
|
||||
{{- else }}
|
||||
@@ -230,13 +225,13 @@ spec:
|
||||
{{- if eq .Values.filer.logs.type "hostPath" }}
|
||||
- name: seaweedfs-filer-log-volume
|
||||
hostPath:
|
||||
path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer
|
||||
path: /storage/logs/seaweedfs/filer
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if eq .Values.filer.data.type "hostPath" }}
|
||||
- name: data-filer
|
||||
hostPath:
|
||||
path: {{ .Values.filer.data.hostPathPrefix }}/filer_store
|
||||
path: /storage/filer_store
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
- name: db-schema-config-volume
|
||||
|
||||
@@ -90,12 +90,7 @@ spec:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
exec /usr/bin/weed \
|
||||
{{- if eq .Values.master.logs.type "hostPath" }}
|
||||
-logdir=/logs \
|
||||
{{- else }}
|
||||
-logtostderr=true \
|
||||
{{- end }}
|
||||
exec /usr/bin/weed -logdir=/logs \
|
||||
{{- if .Values.master.loggingOverrideLevel }}
|
||||
-v={{ .Values.master.loggingOverrideLevel }} \
|
||||
{{- else }}
|
||||
@@ -201,13 +196,13 @@ spec:
|
||||
{{- if eq .Values.master.logs.type "hostPath" }}
|
||||
- name: seaweedfs-master-log-volume
|
||||
hostPath:
|
||||
path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master
|
||||
path: /storage/logs/seaweedfs/master
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if eq .Values.master.data.type "hostPath" }}
|
||||
- name: data-{{ .Release.Namespace }}
|
||||
hostPath:
|
||||
path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/
|
||||
path: /ssd/seaweed-master/
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
|
||||
@@ -72,19 +72,13 @@ spec:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
exec /usr/bin/weed \
|
||||
{{- if eq .Values.s3.logs.type "hostPath" }}
|
||||
-logdir=/logs \
|
||||
{{- else }}
|
||||
-logtostderr=true \
|
||||
{{- end }}
|
||||
exec /usr/bin/weed -logdir=/logs \
|
||||
{{- if .Values.s3.loggingOverrideLevel }}
|
||||
-v={{ .Values.s3.loggingOverrideLevel }} \
|
||||
{{- else }}
|
||||
-v={{ .Values.global.loggingLevel }} \
|
||||
{{- end }}
|
||||
s3 \
|
||||
-ip.bind={{ .Values.s3.bindAddress }} \
|
||||
-port={{ .Values.s3.port }} \
|
||||
{{- if .Values.s3.metricsPort }}
|
||||
-metricsPort {{ .Values.s3.metricsPort }} \
|
||||
@@ -173,7 +167,7 @@ spec:
|
||||
{{- if eq .Values.s3.logs.type "hostPath" }}
|
||||
- name: logs
|
||||
hostPath:
|
||||
path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3
|
||||
path: /storage/logs/seaweedfs/s3
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
|
||||
@@ -34,7 +34,7 @@ spec:
|
||||
restartPolicy: {{ default .Values.global.restartPolicy .Values.volume.restartPolicy }}
|
||||
{{- if .Values.volume.tolerations }}
|
||||
tolerations:
|
||||
{{ tpl .Values.volume.tolerations . | nindent 8 | trim }}
|
||||
{{ tpl .Values.volume.tolerations . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
@@ -93,12 +93,7 @@ spec:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
exec /usr/bin/weed \
|
||||
{{- if eq .Values.volume.logs.type "hostPath" }}
|
||||
-logdir=/logs \
|
||||
{{- else }}
|
||||
-logtostderr=true \
|
||||
{{- end }}
|
||||
exec /usr/bin/weed -logdir=/logs \
|
||||
{{- if .Values.volume.loggingOverrideLevel }}
|
||||
-v={{ .Values.volume.loggingOverrideLevel }} \
|
||||
{{- else }}
|
||||
@@ -211,19 +206,19 @@ spec:
|
||||
{{- if eq .Values.volume.data.type "hostPath" }}
|
||||
- name: data
|
||||
hostPath:
|
||||
path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
|
||||
path: /storage/object_store/
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
|
||||
- name: idx
|
||||
hostPath:
|
||||
path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
|
||||
path: /ssd/seaweedfs-volume-idx/
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if eq .Values.volume.logs.type "hostPath" }}
|
||||
- name: logs
|
||||
hostPath:
|
||||
path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
|
||||
path: /storage/logs/seaweedfs/volume
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
|
||||
@@ -68,13 +68,11 @@ master:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
hostPathPrefix: /ssd
|
||||
|
||||
logs:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
hostPathPrefix: /storage
|
||||
|
||||
initContainers: ""
|
||||
|
||||
@@ -178,19 +176,15 @@ volume:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
hostPathPrefix: /storage
|
||||
|
||||
idx:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
hostPathPrefix: /ssd
|
||||
|
||||
logs:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
hostPathPrefix: /storage
|
||||
|
||||
# limit background compaction or copying speed in mega bytes per second
|
||||
compactionMBps: "50"
|
||||
@@ -314,13 +308,11 @@ filer:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
hostPathPrefix: /storage
|
||||
|
||||
logs:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
hostPathPrefix: /storage
|
||||
|
||||
initContainers: ""
|
||||
|
||||
@@ -426,7 +418,6 @@ s3:
|
||||
imageTag: null
|
||||
restartPolicy: null
|
||||
replicas: 1
|
||||
bindAddress: 0.0.0.0
|
||||
port: 8333
|
||||
metricsPort: 9327
|
||||
loggingOverrideLevel: null
|
||||
@@ -472,7 +463,7 @@ s3:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
hostPathPrefix: /storage
|
||||
|
||||
|
||||
certificates:
|
||||
commonName: "SeaweedFS CA"
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.33</version>
|
||||
<version>3.13</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -14,7 +14,7 @@
|
||||
</parent>
|
||||
|
||||
<properties>
|
||||
<protobuf.version>3.16.3</protobuf.version>
|
||||
<protobuf.version>3.16.1</protobuf.version>
|
||||
<!-- follow https://github.com/grpc/grpc-java -->
|
||||
<grpc.version>1.23.0</grpc.version>
|
||||
<guava.version>30.0-jre</guava.version>
|
||||
@@ -159,7 +159,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.33</version>
|
||||
<version>3.13</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -154,7 +154,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.33</version>
|
||||
<version>3.13</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -128,7 +128,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -83,7 +83,7 @@ public class ReadChunks {
|
||||
prevX,
|
||||
point.x,
|
||||
chunk.getFileId(),
|
||||
chunk.getModifiedTsNs(),
|
||||
chunk.getMtime(),
|
||||
prevX - chunk.getOffset(),
|
||||
chunk.getOffset() == prevX && chunk.getSize() == prevX - startPoint.x,
|
||||
chunk.getCipherKey().toByteArray(),
|
||||
@@ -100,7 +100,7 @@ public class ReadChunks {
|
||||
|
||||
public Point(long x, FilerProto.FileChunk chunk, boolean isStart) {
|
||||
this.x = x;
|
||||
this.ts = chunk.getModifiedTsNs();
|
||||
this.ts = chunk.getMtime();
|
||||
this.chunk = chunk;
|
||||
this.isStart = isStart;
|
||||
}
|
||||
|
||||
@@ -119,8 +119,9 @@ public class SeaweedInputStream extends InputStream {
|
||||
|
||||
long bytesRead = 0;
|
||||
int len = buf.remaining();
|
||||
if (this.position< Integer.MAX_VALUE && (this.position + len )<= entry.getContent().size()) {
|
||||
entry.getContent().substring((int)this.position, (int)(this.position + len)).copyTo(buf);
|
||||
int start = (int) this.position;
|
||||
if (start + len <= entry.getContent().size()) {
|
||||
entry.getContent().substring(start, start + len).copyTo(buf);
|
||||
} else {
|
||||
bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
|
||||
}
|
||||
|
||||
@@ -14,9 +14,7 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.List;
|
||||
import java.util.Base64;
|
||||
|
||||
public class SeaweedWrite {
|
||||
|
||||
@@ -98,7 +96,7 @@ public class SeaweedWrite {
|
||||
.setFileId(fileId)
|
||||
.setOffset(offset)
|
||||
.setSize(bytesLength)
|
||||
.setModifiedTsNs(System.nanoTime())
|
||||
.setMtime(System.currentTimeMillis() / 10000L)
|
||||
.setETag(etag)
|
||||
.setCipherKey(cipherKeyString);
|
||||
}
|
||||
@@ -125,20 +123,13 @@ public class SeaweedWrite {
|
||||
final byte[] bytes,
|
||||
final long bytesOffset, final long bytesLength,
|
||||
byte[] cipherKey) throws IOException {
|
||||
MessageDigest md = null;
|
||||
try {
|
||||
md = MessageDigest.getInstance("MD5");
|
||||
} catch (java.security.NoSuchAlgorithmException e) {
|
||||
}
|
||||
|
||||
InputStream inputStream = null;
|
||||
if (cipherKey == null || cipherKey.length == 0) {
|
||||
md.update(bytes, (int) bytesOffset, (int) bytesLength);
|
||||
inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
|
||||
} else {
|
||||
try {
|
||||
byte[] encryptedBytes = SeaweedCipher.encrypt(bytes, (int) bytesOffset, (int) bytesLength, cipherKey);
|
||||
md.update(encryptedBytes);
|
||||
inputStream = new ByteArrayInputStream(encryptedBytes, 0, encryptedBytes.length);
|
||||
} catch (Exception e) {
|
||||
throw new IOException("fail to encrypt data", e);
|
||||
@@ -149,7 +140,6 @@ public class SeaweedWrite {
|
||||
if (auth != null && auth.length() != 0) {
|
||||
post.addHeader("Authorization", "BEARER " + auth);
|
||||
}
|
||||
post.addHeader("Content-MD5", Base64.getEncoder().encodeToString(md.digest()));
|
||||
|
||||
post.setEntity(MultipartEntityBuilder.create()
|
||||
.setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
|
||||
@@ -159,13 +149,6 @@ public class SeaweedWrite {
|
||||
CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
|
||||
|
||||
try {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
if (response.getEntity().getContentType() != null && response.getEntity().getContentType().getValue().equals("application/json")) {
|
||||
throw new IOException(EntityUtils.toString(response.getEntity(), "UTF-8"));
|
||||
} else {
|
||||
throw new IOException(response.getStatusLine().getReasonPhrase());
|
||||
}
|
||||
}
|
||||
|
||||
String etag = response.getLastHeader("ETag").getValue();
|
||||
|
||||
|
||||
@@ -132,7 +132,7 @@ message FileChunk {
|
||||
string file_id = 1; // to be deprecated
|
||||
int64 offset = 2;
|
||||
uint64 size = 3;
|
||||
int64 modified_ts_ns = 4;
|
||||
int64 mtime = 4;
|
||||
string e_tag = 5;
|
||||
string source_file_id = 6; // to be deprecated
|
||||
FileId fid = 7;
|
||||
|
||||
@@ -17,13 +17,13 @@ public class SeaweedReadTest {
|
||||
.setFileId("aaa")
|
||||
.setOffset(0)
|
||||
.setSize(100)
|
||||
.setModifiedTsNs(1000)
|
||||
.setMtime(1000)
|
||||
.build());
|
||||
chunks.add(FilerProto.FileChunk.newBuilder()
|
||||
.setFileId("bbb")
|
||||
.setOffset(100)
|
||||
.setSize(133)
|
||||
.setModifiedTsNs(2000)
|
||||
.setMtime(2000)
|
||||
.build());
|
||||
|
||||
List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(null, chunks);
|
||||
@@ -70,31 +70,31 @@ public class SeaweedReadTest {
|
||||
.setFileId("a")
|
||||
.setOffset(0)
|
||||
.setSize(100)
|
||||
.setModifiedTsNs(1)
|
||||
.setMtime(1)
|
||||
.build());
|
||||
chunks.add(FilerProto.FileChunk.newBuilder()
|
||||
.setFileId("b")
|
||||
.setOffset(50)
|
||||
.setSize(100)
|
||||
.setModifiedTsNs(2)
|
||||
.setMtime(2)
|
||||
.build());
|
||||
chunks.add(FilerProto.FileChunk.newBuilder()
|
||||
.setFileId("c")
|
||||
.setOffset(200)
|
||||
.setSize(50)
|
||||
.setModifiedTsNs(3)
|
||||
.setMtime(3)
|
||||
.build());
|
||||
chunks.add(FilerProto.FileChunk.newBuilder()
|
||||
.setFileId("d")
|
||||
.setOffset(250)
|
||||
.setSize(50)
|
||||
.setModifiedTsNs(4)
|
||||
.setMtime(4)
|
||||
.build());
|
||||
chunks.add(FilerProto.FileChunk.newBuilder()
|
||||
.setFileId("e")
|
||||
.setOffset(175)
|
||||
.setSize(100)
|
||||
.setModifiedTsNs(5)
|
||||
.setMtime(5)
|
||||
.build());
|
||||
|
||||
List<SeaweedRead.VisibleInterval> visibleIntervals = ReadChunks.readResolvedChunks(chunks);
|
||||
@@ -161,7 +161,7 @@ public class SeaweedReadTest {
|
||||
.setFileId("")
|
||||
.setOffset(start)
|
||||
.setSize(size)
|
||||
.setModifiedTsNs(ts)
|
||||
.setMtime(ts)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,13 +11,13 @@
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.33</version>
|
||||
<version>3.13</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-hadoop2-client</artifactId>
|
||||
<version>3.33</version>
|
||||
<version>3.13</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
<dependency>
|
||||
<groupId>io.springfox</groupId>
|
||||
<artifactId>springfox-swagger-ui</artifactId>
|
||||
<version>2.10.0</version>
|
||||
<version>2.9.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
@@ -124,7 +124,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>3.2.4</version>
|
||||
<version>2.10.1</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -132,11 +132,11 @@
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
<artifactId>hadoop-mapreduce-client-app</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-yarn-client</artifactId>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
@@ -156,7 +156,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>3.2.4</version>
|
||||
<version>2.10.1</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -167,6 +167,10 @@
|
||||
<artifactId>commons-math3</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>xmlenc</artifactId>
|
||||
<groupId>xmlenc</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<groupId>commons-io</groupId>
|
||||
@@ -180,28 +184,20 @@
|
||||
<groupId>commons-collections</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
<groupId>javax.servlet</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>javax.activation-api</artifactId>
|
||||
<groupId>javax.activation</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty</artifactId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-util</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-webapp</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-sslengine</artifactId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jsp-api</artifactId>
|
||||
@@ -211,10 +207,6 @@
|
||||
<artifactId>jersey-core</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-servlet</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-json</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
@@ -224,37 +216,49 @@
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>reload4j</artifactId>
|
||||
<groupId>ch.qos.reload4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jets3t</artifactId>
|
||||
<groupId>net.java.dev.jets3t</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
<groupId>commons-lang</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-configuration</artifactId>
|
||||
<groupId>commons-configuration</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-digester</artifactId>
|
||||
<groupId>commons-digester</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
<groupId>commons-beanutils</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-configuration2</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-text</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-reload4j</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>jackson-core-asl</artifactId>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>avro</artifactId>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>re2j</artifactId>
|
||||
<groupId>com.google.re2j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-auth</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
@@ -283,14 +287,6 @@
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>kerb-simplekdc</artifactId>
|
||||
<groupId>org.apache.kerby</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>stax2-api</artifactId>
|
||||
<groupId>org.codehaus.woodstox</groupId>
|
||||
@@ -299,10 +295,6 @@
|
||||
<artifactId>woodstox-core</artifactId>
|
||||
<groupId>com.fasterxml.woodstox</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>dnsjava</artifactId>
|
||||
<groupId>dnsjava</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-annotations</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
@@ -317,7 +309,7 @@
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.33</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>2.10.1</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.33</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
|
||||
@@ -129,7 +129,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
@@ -124,7 +124,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>3.2.4</version>
|
||||
<version>3.2.3</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -156,7 +156,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>3.2.4</version>
|
||||
<version>3.2.3</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -224,8 +224,8 @@
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>reload4j</artifactId>
|
||||
<groupId>ch.qos.reload4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
@@ -244,7 +244,7 @@
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-reload4j</artifactId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
@@ -317,7 +317,7 @@
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.33</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.3</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.33</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
|
||||
@@ -129,7 +129,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -91,7 +91,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",
|
||||
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@@ -100,7 +100,7 @@
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",
|
||||
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@@ -181,7 +181,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@@ -190,7 +190,7 @@
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@@ -277,7 +277,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@@ -286,7 +286,7 @@
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@@ -386,7 +386,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(SeaweedFS_filerStore_request_total[1m]) * 5",
|
||||
"expr": "rate(SeaweedFS_filer_request_total[1m]) * 5",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{type}}",
|
||||
@@ -1929,4 +1929,4 @@
|
||||
"timezone": "browser",
|
||||
"title": "SeaweedFS ks8 heartbeat",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"strconv"
|
||||
@@ -53,7 +52,7 @@ func main() {
|
||||
}
|
||||
|
||||
func startGenerateMetadata() {
|
||||
pb.WithFilerClient(false, util.RandomInt32(), pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
|
||||
pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
for i := 0; i < *n; i++ {
|
||||
name := fmt.Sprintf("file%d", i)
|
||||
|
||||
@@ -59,7 +59,7 @@ func walkMetaFile(dst *os.File) error {
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stdout, "file %s %v\n", util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name), fullEntry.Entry.Attributes.String())
|
||||
for i, chunk := range fullEntry.Entry.GetChunks() {
|
||||
for i, chunk := range fullEntry.Entry.Chunks {
|
||||
fmt.Fprintf(os.Stdout, " chunk: %d %v %d,%x%08x\n", i+1, chunk, chunk.Fid.VolumeId, chunk.Fid.FileKey, chunk.Fid.Cookie)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,14 +5,13 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"io"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -30,7 +29,7 @@ func main() {
|
||||
vid := uint32(*volumeId)
|
||||
|
||||
eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
|
||||
fmt.Printf("%d,%x%08x %d %v %d %x\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob), resp.NeedleBlobCompressed, resp.LastModified, resp.Crc)
|
||||
fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+11
-11
@@ -15,44 +15,44 @@ clean:
|
||||
|
||||
debug_shell:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- shell
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- shell
|
||||
|
||||
debug_mount:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000
|
||||
|
||||
debug_server:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
|
||||
|
||||
debug_volume:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- volume -dir=~/tmp/100 -port 8564 -max=30 -preStopSeconds=2
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- volume -dir=~/tmp/100 -port 8564 -max=30 -preStopSeconds=2
|
||||
|
||||
debug_webdav:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 webdav
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 webdav
|
||||
|
||||
debug_s3:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 s3
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 s3
|
||||
|
||||
debug_filer_copy:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h
|
||||
|
||||
debug_filer_remote_sync_dir:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.remote.sync -filer="localhost:8888" -dir=/buckets/b2 -timeAgo=1h
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -dir=/buckets/b2 -timeAgo=1h
|
||||
|
||||
debug_filer_remote_sync_buckets:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.remote.sync -filer="localhost:8888" -createBucketAt=cloud1 -timeAgo=1h
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -createBucketAt=cloud1 -timeAgo=1h
|
||||
|
||||
debug_master_follower:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 master.follower
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 master.follower
|
||||
|
||||
debug_filer_sync:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.sync -a 192.168.2.7:8888 -b 192.168.2.7:8889 -isActivePassive -b.debug
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.sync -a 192.168.2.7:8888 -b 192.168.2.7:8889 -isActivePassive -b.debug
|
||||
|
||||
@@ -2,7 +2,6 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
@@ -114,7 +113,7 @@ func runBackup(cmd *Command, args []string) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
|
||||
v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
|
||||
if err != nil {
|
||||
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
|
||||
return true
|
||||
@@ -139,7 +138,7 @@ func runBackup(cmd *Command, args []string) bool {
|
||||
// remove the old data
|
||||
v.Destroy()
|
||||
// recreate an empty volume
|
||||
v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
|
||||
v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
|
||||
if err != nil {
|
||||
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
|
||||
return true
|
||||
|
||||
@@ -41,6 +41,7 @@ type BenchmarkOptions struct {
|
||||
grpcDialOption grpc.DialOption
|
||||
masterClient *wdclient.MasterClient
|
||||
fsync *bool
|
||||
useTcp *bool
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -67,6 +68,7 @@ func init() {
|
||||
b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
|
||||
b.fsync = cmdBenchmark.Flag.Bool("fsync", false, "flush data to disk after write")
|
||||
b.useTcp = cmdBenchmark.Flag.Bool("useTcp", false, "send data via tcp")
|
||||
sharedBytes = make([]byte, 1024)
|
||||
}
|
||||
|
||||
@@ -223,6 +225,8 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
|
||||
|
||||
random := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
volumeTcpClient := wdclient.NewVolumeTcpClient()
|
||||
|
||||
for id := range idChan {
|
||||
start := time.Now()
|
||||
fileSize := int64(*b.fileSize + random.Intn(64))
|
||||
@@ -243,7 +247,15 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
|
||||
if !isSecure && assignResult.Auth != "" {
|
||||
isSecure = true
|
||||
}
|
||||
if _, err := fp.Upload(0, b.masterClient.GetMaster, false, assignResult.Auth, b.grpcDialOption); err == nil {
|
||||
if *b.useTcp {
|
||||
if uploadByTcp(volumeTcpClient, fp) {
|
||||
fileIdLineChan <- fp.Fid
|
||||
s.completed++
|
||||
s.transferred += fileSize
|
||||
} else {
|
||||
s.failed++
|
||||
}
|
||||
} else if _, err := fp.Upload(0, b.masterClient.GetMaster, false, assignResult.Auth, b.grpcDialOption); err == nil {
|
||||
if random.Intn(100) < *b.deletePercentage {
|
||||
s.total++
|
||||
delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
|
||||
@@ -329,6 +341,17 @@ func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan b
|
||||
}
|
||||
}
|
||||
|
||||
func uploadByTcp(volumeTcpClient *wdclient.VolumeTcpClient, fp *operation.FilePart) bool {
|
||||
|
||||
err := volumeTcpClient.PutFileChunk(fp.Server, fp.Fid, uint32(fp.FileSize), fp.Reader)
|
||||
if err != nil {
|
||||
glog.Errorf("upload chunk err: %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func readFileIds(fileName string, fileIdLineChan chan string) {
|
||||
file, err := os.Open(fileName) // For read access.
|
||||
if err != nil {
|
||||
|
||||
@@ -41,7 +41,7 @@ func runCompact(cmd *Command, args []string) bool {
|
||||
preallocate := *compactVolumePreallocate * (1 << 20)
|
||||
|
||||
vid := needle.VolumeId(*compactVolumeId)
|
||||
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, 0, 0)
|
||||
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, 0)
|
||||
if err != nil {
|
||||
glog.Fatalf("Load Volume [ERROR] %s\n", err)
|
||||
}
|
||||
|
||||
@@ -113,7 +113,6 @@ func init() {
|
||||
filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
|
||||
filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
|
||||
filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
|
||||
filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
|
||||
|
||||
// start iam on filer
|
||||
filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
|
||||
@@ -163,7 +162,7 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
|
||||
util.LoadConfiguration("security", false)
|
||||
|
||||
go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort)
|
||||
go stats_collect.StartMetricsServer(*f.metricsHttpPort)
|
||||
|
||||
filerAddress := util.JoinHostPort(*f.ip, *f.port)
|
||||
startDelay := time.Duration(2)
|
||||
@@ -299,7 +298,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
|
||||
}
|
||||
if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
|
||||
glog.Fatalf("Failed to remove %s, error: %v", localSocket, err)
|
||||
}
|
||||
go func() {
|
||||
// start on local unix socket
|
||||
|
||||
@@ -19,7 +19,6 @@ type FilerBackupOptions struct {
|
||||
debug *bool
|
||||
proxyByFiler *bool
|
||||
timeAgo *time.Duration
|
||||
retentionDays *int
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -34,8 +33,6 @@ func init() {
|
||||
filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers")
|
||||
filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files")
|
||||
filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
|
||||
filerBackupOptions.retentionDays = cmdFilerBackup.Flag.Int("retentionDays", 0, "incremental backup retention days")
|
||||
|
||||
}
|
||||
|
||||
var cmdFilerBackup = &Command{
|
||||
@@ -126,18 +123,6 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
|
||||
return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs)
|
||||
})
|
||||
|
||||
if dataSink.IsIncremental() && *filerBackupOptions.retentionDays > 0 {
|
||||
go func() {
|
||||
for {
|
||||
now := time.Now()
|
||||
time.Sleep(time.Hour * 24)
|
||||
key := util.Join(targetPath, now.Add(-1*time.Hour*24*time.Duration(*filerBackupOptions.retentionDays)).Format("2006-01-02"))
|
||||
_ = dataSink.DeleteEntry(util.Join(targetPath, key), true, true, nil)
|
||||
glog.V(0).Infof("incremental backup delete directory:%s", key)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, clientEpoch, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ func runFilerCat(cmd *Command, args []string) bool {
|
||||
writer = f
|
||||
}
|
||||
|
||||
pb.WithFilerClient(false, util.RandomInt32(), filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
pb.WithFilerClient(false, filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
request := &filer_pb.LookupDirectoryEntryRequest{
|
||||
Name: name,
|
||||
@@ -114,7 +114,7 @@ func runFilerCat(cmd *Command, args []string) bool {
|
||||
|
||||
filerCat.filerClient = client
|
||||
|
||||
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.GetChunks(), 0, int64(filer.FileSize(respLookupEntry.Entry)))
|
||||
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry)))
|
||||
|
||||
})
|
||||
|
||||
|
||||
@@ -159,7 +159,6 @@ func runCopy(cmd *Command, args []string) bool {
|
||||
worker := FileCopyWorker{
|
||||
options: ©,
|
||||
filerAddress: filerAddress,
|
||||
signature: util.RandomInt32(),
|
||||
}
|
||||
if err := worker.copyFiles(fileCopyTaskChan); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
|
||||
@@ -173,7 +172,7 @@ func runCopy(cmd *Command, args []string) bool {
|
||||
}
|
||||
|
||||
func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress pb.ServerAddress) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
|
||||
err = pb.WithGrpcFilerClient(false, 0, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
err = pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
|
||||
@@ -226,7 +225,6 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
|
||||
type FileCopyWorker struct {
|
||||
options *CopyOptions
|
||||
filerAddress pb.ServerAddress
|
||||
signature int32
|
||||
}
|
||||
|
||||
func (worker *FileCopyWorker) copyFiles(fileCopyTaskChan chan FileCopyTask) error {
|
||||
@@ -304,7 +302,7 @@ func (worker *FileCopyWorker) checkExistingFileFirst(task FileCopyTask, f *os.Fi
|
||||
return
|
||||
}
|
||||
|
||||
err = pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
err = pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
request := &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: task.destinationUrlPath,
|
||||
@@ -367,10 +365,10 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
|
||||
if flushErr != nil {
|
||||
return flushErr
|
||||
}
|
||||
chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0, time.Now().UnixNano()))
|
||||
chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0))
|
||||
}
|
||||
|
||||
if err := pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
request := &filer_pb.CreateEntryRequest{
|
||||
Directory: task.destinationUrlPath,
|
||||
Entry: &filer_pb.Entry{
|
||||
@@ -452,7 +450,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
|
||||
uploadError = fmt.Errorf("upload %v result: %v\n", fileName, uploadResult.Error)
|
||||
return
|
||||
}
|
||||
chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize, time.Now().UnixNano())
|
||||
chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize)
|
||||
|
||||
fmt.Printf("uploaded %s-%d [%d,%d)\n", fileName, i+1, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
|
||||
}(i)
|
||||
@@ -481,7 +479,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
|
||||
return fmt.Errorf("create manifest: %v", manifestErr)
|
||||
}
|
||||
|
||||
if err := pb.WithGrpcFilerClient(false, worker.signature, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
request := &filer_pb.CreateEntryRequest{
|
||||
Directory: task.destinationUrlPath,
|
||||
Entry: &filer_pb.Entry{
|
||||
@@ -532,7 +530,7 @@ func detectMimeType(f *os.File) string {
|
||||
return mimeType
|
||||
}
|
||||
|
||||
func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error) {
|
||||
func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, err error) {
|
||||
|
||||
finalFileId, uploadResult, flushErr, _ := operation.UploadWithRetry(
|
||||
worker,
|
||||
@@ -563,7 +561,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
|
||||
if uploadResult.Error != "" {
|
||||
return nil, fmt.Errorf("upload result: %v", uploadResult.Error)
|
||||
}
|
||||
return uploadResult.ToPbFileChunk(finalFileId, offset, tsNs), nil
|
||||
return uploadResult.ToPbFileChunk(finalFileId, offset), nil
|
||||
}
|
||||
|
||||
var _ = filer_pb.FilerClient(&FileCopyWorker{})
|
||||
@@ -571,7 +569,7 @@ var _ = filer_pb.FilerClient(&FileCopyWorker{})
|
||||
func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
|
||||
|
||||
filerGrpcAddress := worker.filerAddress.ToGrpcAddress()
|
||||
err = pb.WithGrpcClient(streamingMode, worker.signature, func(grpcConnection *grpc.ClientConn) error {
|
||||
err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
|
||||
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
|
||||
return fn(client)
|
||||
}, filerGrpcAddress, false, worker.options.grpcDialOption)
|
||||
|
||||
@@ -225,7 +225,7 @@ var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{})
|
||||
|
||||
func (metaBackup *FilerMetaBackupOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
|
||||
return pb.WithFilerClient(streamingMode, metaBackup.clientId, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return pb.WithFilerClient(streamingMode, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return fn(client)
|
||||
})
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ type RemoteGatewayOptions struct {
|
||||
var _ = filer_pb.FilerClient(&RemoteGatewayOptions{})
|
||||
|
||||
func (option *RemoteGatewayOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
return pb.WithFilerClient(streamingMode, option.clientId, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return fn(client)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ type RemoteSyncOptions struct {
|
||||
var _ = filer_pb.FilerClient(&RemoteSyncOptions{})
|
||||
|
||||
func (option *RemoteSyncOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
return pb.WithFilerClient(streamingMode, option.clientId, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return fn(client)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -37,16 +37,11 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
|
||||
|
||||
var lastLogTsNs = time.Now().UnixNano()
|
||||
processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if resp.EventNotification.NewEntry != nil {
|
||||
if *option.storageClass == "" {
|
||||
if _, ok := resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass]; ok {
|
||||
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
|
||||
}
|
||||
} else {
|
||||
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
|
||||
}
|
||||
if *option.storageClass == "" {
|
||||
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
|
||||
} else {
|
||||
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
|
||||
}
|
||||
|
||||
processor.AddSyncJob(resp)
|
||||
return nil
|
||||
}, 3*time.Second, func(counter int64, lastTsNs int64) error {
|
||||
|
||||
@@ -43,7 +43,6 @@ type SyncOptions struct {
|
||||
bFromTsMs *int64
|
||||
aProxyByFiler *bool
|
||||
bProxyByFiler *bool
|
||||
metricsHttpIp *string
|
||||
metricsHttpPort *int
|
||||
concurrency *int
|
||||
clientId int32
|
||||
@@ -87,7 +86,6 @@ func init() {
|
||||
syncOptions.concurrency = cmdFilerSynchronize.Flag.Int("concurrency", DefaultConcurrencyLimit, "The maximum number of files that will be synced concurrently.")
|
||||
syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
syncMemProfile = cmdFilerSynchronize.Flag.String("memprofile", "", "memory profile output file")
|
||||
syncOptions.metricsHttpIp = cmdFilerSynchronize.Flag.String("metricsIp", "", "metrics listen ip")
|
||||
syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
|
||||
syncOptions.clientId = util.RandomInt32()
|
||||
}
|
||||
@@ -121,7 +119,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
|
||||
filerB := pb.ServerAddress(*syncOptions.filerB)
|
||||
|
||||
// start filer.sync metrics server
|
||||
go statsCollect.StartMetricsServer(*syncOptions.metricsHttpIp, *syncOptions.metricsHttpPort)
|
||||
go statsCollect.StartMetricsServer(*syncOptions.metricsHttpPort)
|
||||
|
||||
// read a filer signature
|
||||
aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
|
||||
@@ -304,7 +302,7 @@ func getSignaturePrefixByPath(path string) string {
|
||||
|
||||
func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
|
||||
|
||||
readErr = pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
syncKey := []byte(signaturePrefix + "____")
|
||||
util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature))
|
||||
|
||||
@@ -330,7 +328,7 @@ func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature
|
||||
}
|
||||
|
||||
func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error {
|
||||
return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
syncKey := []byte(signaturePrefix + "____")
|
||||
util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature))
|
||||
@@ -399,11 +397,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
|
||||
return nil
|
||||
}
|
||||
key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
|
||||
if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
|
||||
return fmt.Errorf("create entry1 : %v", err)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
|
||||
}
|
||||
|
||||
// this is something special?
|
||||
@@ -431,11 +425,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
|
||||
}
|
||||
// create the new entry
|
||||
newKey := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
|
||||
if err := dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures); err != nil {
|
||||
return fmt.Errorf("create entry2 : %v", err)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
return dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures)
|
||||
|
||||
} else {
|
||||
// new key is outside of the watched directory
|
||||
@@ -449,11 +439,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
|
||||
if strings.HasPrefix(string(sourceNewKey), sourcePath) {
|
||||
// new key is in the watched directory
|
||||
key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
|
||||
if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
|
||||
return fmt.Errorf("create entry3 : %v", err)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
|
||||
} else {
|
||||
// new key is also outside of the watched directory
|
||||
// skip
|
||||
|
||||
+2
-13
@@ -31,7 +31,6 @@ var cmdFix = &Command{
|
||||
var (
|
||||
fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
|
||||
fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
|
||||
fixIgnoreError = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
|
||||
)
|
||||
|
||||
type VolumeFileScanner4Fix struct {
|
||||
@@ -127,21 +126,11 @@ func doFixOneVolume(basepath string, baseFileName string, collection string, vol
|
||||
}
|
||||
|
||||
if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
|
||||
err := fmt.Errorf("scan .dat File: %v", err)
|
||||
if *fixIgnoreError {
|
||||
glog.Error(err)
|
||||
} else {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
glog.Fatalf("scan .dat File: %v", err)
|
||||
}
|
||||
|
||||
if err := nm.SaveToIdx(indexFileName); err != nil {
|
||||
os.Remove(indexFileName)
|
||||
err := fmt.Errorf("save to .idx File: %v", err)
|
||||
if *fixIgnoreError {
|
||||
glog.Error(err)
|
||||
} else {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
glog.Fatalf("save to .idx File: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-10
@@ -47,10 +47,7 @@ func runFuse(cmd *Command, args []string) bool {
|
||||
for i++; i < rawArgsLen && rawArgs[i] != ' '; i++ {
|
||||
option.WriteByte(rawArgs[i])
|
||||
}
|
||||
// ignore "-o"
|
||||
if option.String() != "o" {
|
||||
options = append(options, parameter{option.String(), "true"})
|
||||
}
|
||||
options = append(options, parameter{option.String(), "true"})
|
||||
option.Reset()
|
||||
|
||||
// equal separator start option with pending value
|
||||
@@ -193,12 +190,6 @@ func runFuse(cmd *Command, args []string) bool {
|
||||
}
|
||||
case "fusermount.path":
|
||||
fusermountPath = parameter.value
|
||||
default:
|
||||
t := parameter.name
|
||||
if parameter.value != "true" {
|
||||
t = fmt.Sprintf("%s=%s", parameter.name, parameter.value)
|
||||
}
|
||||
mountOptions.extraOptions = append(mountOptions.extraOptions, t)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@ func (iamopt *IamOptions) startIamServer() bool {
|
||||
util.LoadConfiguration("security", false)
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
|
||||
for {
|
||||
err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
|
||||
|
||||
@@ -2,7 +2,6 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@@ -119,7 +118,7 @@ func runMaster(cmd *Command, args []string) bool {
|
||||
glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
|
||||
}
|
||||
|
||||
go stats_collect.StartMetricsServer(*m.ipBind, *m.metricsHttpPort)
|
||||
go stats_collect.StartMetricsServer(*m.metricsHttpPort)
|
||||
startMaster(m, masterWhiteList)
|
||||
|
||||
return true
|
||||
@@ -254,13 +253,6 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
go httpS.Serve(masterListener)
|
||||
}
|
||||
|
||||
grace.OnInterrupt(ms.Shutdown)
|
||||
grace.OnInterrupt(grpcS.Stop)
|
||||
grace.OnReload(func() {
|
||||
if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
|
||||
ms.Topo.HashicorpRaft.LeadershipTransfer()
|
||||
}
|
||||
})
|
||||
select {}
|
||||
}
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ type MountOptions struct {
|
||||
debugPort *int
|
||||
localSocket *string
|
||||
disableXAttr *bool
|
||||
extraOptions []string
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -53,9 +52,9 @@ func init() {
|
||||
mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
|
||||
mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
|
||||
mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers")
|
||||
mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
|
||||
mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
|
||||
mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "file chunk read cache capacity in MB")
|
||||
mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB")
|
||||
mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
|
||||
mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
|
||||
mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
|
||||
|
||||
@@ -166,7 +166,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
// mount fuse
|
||||
fuseMountOptions := &fuse.MountOptions{
|
||||
AllowOther: *option.allowOthers,
|
||||
Options: option.extraOptions,
|
||||
Options: nil,
|
||||
MaxBackground: 128,
|
||||
MaxWrite: 1024 * 1024 * 2,
|
||||
MaxReadAhead: 1024 * 1024 * 2,
|
||||
@@ -256,8 +256,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
|
||||
seaweedFileSystem.StartBackgroundTasks()
|
||||
|
||||
glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir)
|
||||
glog.V(0).Infof("This is SeaweedFS version %s %s %s", util.Version(), runtime.GOOS, runtime.GOARCH)
|
||||
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
|
||||
|
||||
server.Serve()
|
||||
|
||||
|
||||
+2
-2
@@ -144,7 +144,7 @@ func runS3(cmd *Command, args []string) bool {
|
||||
|
||||
util.LoadConfiguration("security", false)
|
||||
|
||||
go stats_collect.StartMetricsServer(*s3StandaloneOptions.bindIp, *s3StandaloneOptions.metricsHttpPort)
|
||||
go stats_collect.StartMetricsServer(*s3StandaloneOptions.metricsHttpPort)
|
||||
|
||||
return s3StandaloneOptions.startS3Server()
|
||||
|
||||
@@ -163,7 +163,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
var metricsIntervalSec int
|
||||
|
||||
for {
|
||||
err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
|
||||
|
||||
@@ -41,13 +41,13 @@ enabled = false
|
||||
dbFile = "./filer.db" # sqlite db file
|
||||
|
||||
[mysql] # or memsql, tidb
|
||||
# CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
# `dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
# `name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
# `directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
# `meta` LONGBLOB,
|
||||
# PRIMARY KEY (`dirhash`, `name`)
|
||||
# ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
# CREATE TABLE IF NOT EXISTS filemeta (
|
||||
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
# name VARCHAR(1000) BINARY COMMENT 'directory or file name',
|
||||
# directory TEXT BINARY COMMENT 'full path to parent directory',
|
||||
# meta LONGBLOB,
|
||||
# PRIMARY KEY (dirhash, name)
|
||||
# ) DEFAULT CHARSET=utf8;
|
||||
|
||||
enabled = false
|
||||
hostname = "localhost"
|
||||
@@ -61,18 +61,18 @@ connection_max_lifetime_seconds = 0
|
||||
interpolateParams = false
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
|
||||
upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
|
||||
|
||||
[mysql2] # or memsql, tidb
|
||||
enabled = false
|
||||
createTable = """
|
||||
CREATE TABLE IF NOT EXISTS `%s` (
|
||||
`dirhash` BIGINT NOT NULL,
|
||||
`name` VARCHAR(766) NOT NULL,
|
||||
`directory` TEXT NOT NULL,
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
dirhash BIGINT,
|
||||
name VARCHAR(1000) BINARY,
|
||||
directory TEXT BINARY,
|
||||
meta LONGBLOB,
|
||||
PRIMARY KEY (dirhash, name)
|
||||
) DEFAULT CHARSET=utf8;
|
||||
"""
|
||||
hostname = "localhost"
|
||||
port = 3306
|
||||
@@ -85,7 +85,7 @@ connection_max_lifetime_seconds = 0
|
||||
interpolateParams = false
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
|
||||
upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
|
||||
|
||||
[postgres] # or cockroachdb, YugabyteDB
|
||||
# CREATE TABLE IF NOT EXISTS filemeta (
|
||||
|
||||
@@ -68,7 +68,6 @@ is_incremental = false
|
||||
enabled = false
|
||||
b2_account_id = ""
|
||||
b2_master_application_key = ""
|
||||
b2_region = ""
|
||||
bucket = "mybucket" # an existing bucket
|
||||
directory = "/" # destination directory
|
||||
is_incremental = false
|
||||
|
||||
@@ -92,11 +92,10 @@ func init() {
|
||||
masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
|
||||
masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
|
||||
masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
|
||||
masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
|
||||
masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
|
||||
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
|
||||
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
|
||||
masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
|
||||
masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
|
||||
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server")
|
||||
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
|
||||
@@ -125,7 +124,6 @@ func init() {
|
||||
serverOptions.v.readMode = cmdServer.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
|
||||
serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
|
||||
serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
serverOptions.v.ldbTimeout = cmdServer.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
|
||||
serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
|
||||
serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 64, "limit total concurrent download size")
|
||||
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
|
||||
@@ -133,7 +131,7 @@ func init() {
|
||||
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
|
||||
serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
|
||||
serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
|
||||
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", false, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally")
|
||||
|
||||
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
|
||||
@@ -156,7 +154,6 @@ func init() {
|
||||
webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
|
||||
webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
|
||||
webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
|
||||
webdavOptions.filerRootPath = cmdServer.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
|
||||
|
||||
mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port")
|
||||
|
||||
@@ -234,7 +231,7 @@ func runServer(cmd *Command, args []string) bool {
|
||||
webdavOptions.filer = &filerAddress
|
||||
mqBrokerOptions.filerGroup = filerOptions.filerGroup
|
||||
|
||||
go stats_collect.StartMetricsServer(*serverBindIp, *serverMetricsHttpPort)
|
||||
go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
|
||||
|
||||
folders := strings.Split(*volumeDataFolders, ",")
|
||||
|
||||
|
||||
@@ -68,7 +68,6 @@ type VolumeServerOptions struct {
|
||||
inflightUploadDataTimeout *time.Duration
|
||||
hasSlowRead *bool
|
||||
readBufferSizeMB *int
|
||||
ldbTimeout *int64
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -93,14 +92,13 @@ func init() {
|
||||
v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
|
||||
v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
|
||||
v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
v.ldbTimeout = cmdVolume.Flag.Int64("index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
|
||||
v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 256, "limit total concurrent upload size")
|
||||
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 256, "limit total concurrent download size")
|
||||
v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
|
||||
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
|
||||
v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
|
||||
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", false, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
|
||||
}
|
||||
|
||||
@@ -130,7 +128,7 @@ func runVolume(cmd *Command, args []string) bool {
|
||||
grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
|
||||
}
|
||||
|
||||
go stats_collect.StartMetricsServer(*v.bindIp, *v.metricsHttpPort)
|
||||
go stats_collect.StartMetricsServer(*v.metricsHttpPort)
|
||||
|
||||
minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
|
||||
v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
|
||||
@@ -251,7 +249,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
*v.inflightUploadDataTimeout,
|
||||
*v.hasSlowRead,
|
||||
*v.readBufferSizeMB,
|
||||
*v.ldbTimeout,
|
||||
)
|
||||
// starting grpc server
|
||||
grpcS := v.startGrpcService(volumeServer)
|
||||
@@ -268,8 +265,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
// starting the cluster http server
|
||||
clusterHttpServer := v.startClusterHttpService(volumeMux)
|
||||
|
||||
grace.OnReload(volumeServer.LoadNewVolumes)
|
||||
|
||||
stopChan := make(chan bool)
|
||||
grace.OnInterrupt(func() {
|
||||
fmt.Println("volume server has been killed")
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
|
||||
"github.com/seaweedfs/seaweedfs/weed/server"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
@@ -23,7 +23,6 @@ var (
|
||||
|
||||
type WebDavOption struct {
|
||||
filer *string
|
||||
filerRootPath *string
|
||||
port *int
|
||||
collection *string
|
||||
replication *string
|
||||
@@ -45,7 +44,6 @@ func init() {
|
||||
webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
|
||||
webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
|
||||
webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB")
|
||||
webDavStandaloneOptions.filerRootPath = cmdWebDav.Flag.String("filer.path", "/", "use this remote path from filer server")
|
||||
}
|
||||
|
||||
var cmdWebDav = &Command{
|
||||
@@ -87,7 +85,7 @@ func (wo *WebDavOption) startWebDav() bool {
|
||||
var cipher bool
|
||||
// connect to filer
|
||||
for {
|
||||
err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
|
||||
@@ -106,7 +104,6 @@ func (wo *WebDavOption) startWebDav() bool {
|
||||
|
||||
ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
|
||||
Filer: filerAddress,
|
||||
FilerRootPath: *wo.filerRootPath,
|
||||
GrpcDialOption: grpcDialOption,
|
||||
Collection: *wo.collection,
|
||||
Replication: *wo.replication,
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
all: gen
|
||||
|
||||
.PHONY : gen
|
||||
|
||||
gen:
|
||||
protoc columnar.proto --go_out=./columnar_pb --go-grpc_out=./columnar_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
|
||||
@@ -1,32 +0,0 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
type ColumnUint16 struct {
|
||||
}
|
||||
|
||||
const SIZE_Uint16 = 2
|
||||
|
||||
func (c *ColumnUint16) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint16 {
|
||||
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
|
||||
return binary.BigEndian.Uint16(buf)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func WriteUint16s(buf []byte, data []uint16) (err error) {
|
||||
off := 0
|
||||
size := len(data)
|
||||
if len(buf) < size<<1 {
|
||||
return fmt.Errorf("buf too small")
|
||||
}
|
||||
for _, dat := range data {
|
||||
binary.BigEndian.PutUint16(buf[off:], dat)
|
||||
off += SIZE_Uint16
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
type ColumnUint32 struct {
|
||||
}
|
||||
|
||||
const SIZE_Uint32 = 4
|
||||
|
||||
func (c *ColumnUint32) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint32 {
|
||||
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
|
||||
return binary.BigEndian.Uint32(buf)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func WriteUint32s(buf []byte, data []uint32) (err error) {
|
||||
off := 0
|
||||
size := len(data)
|
||||
if len(buf) < size<<2 {
|
||||
return fmt.Errorf("buf too small")
|
||||
}
|
||||
for _, dat := range data {
|
||||
binary.BigEndian.PutUint32(buf[off:], dat)
|
||||
off += SIZE_Uint32
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
type ColumnUint64 struct {
|
||||
}
|
||||
|
||||
const SIZE_Uint64 = 8
|
||||
|
||||
func (c *ColumnUint64) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint64 {
|
||||
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint64); n == SIZE_Uint64 && err == nil {
|
||||
return binary.BigEndian.Uint64(buf)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func WriteUint64s(buf []byte, data []uint64) (err error) {
|
||||
off := 0
|
||||
size := len(data)
|
||||
if len(buf) < size<<3 {
|
||||
return fmt.Errorf("buf too small")
|
||||
}
|
||||
for _, dat := range data {
|
||||
binary.BigEndian.PutUint64(buf[off:], dat)
|
||||
off += SIZE_Uint64
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package columnar_pb;
|
||||
|
||||
option go_package = "github.com/seaweedfs/seaweedfs/weed/data/columnar_pb";
|
||||
|
||||
message FileId {
|
||||
uint32 volume_id = 1;
|
||||
uint64 file_key = 2;
|
||||
fixed32 cookie = 3;
|
||||
}
|
||||
|
||||
enum LogicalType {
|
||||
Uint8 = 0;
|
||||
Uint16 = 1;
|
||||
Float32 = 4;
|
||||
}
|
||||
|
||||
message ColumnUint16 {
|
||||
uint32 base = 1;
|
||||
uint32 min = 3;
|
||||
uint32 max = 4;
|
||||
}
|
||||
|
||||
message ColumnUint32 {
|
||||
uint32 base = 1;
|
||||
uint32 min = 3;
|
||||
uint32 max = 4;
|
||||
}
|
||||
|
||||
message ColumnFloat32 {
|
||||
uint32 min = 3;
|
||||
uint32 max = 4;
|
||||
}
|
||||
|
||||
message ColumnSplit {
|
||||
// The ids of the fields/columns in this file
|
||||
int32 field_id = 1;
|
||||
FileId file_id = 2;
|
||||
int64 row_offset = 3;
|
||||
int32 row_count = 4;
|
||||
|
||||
oneof storage_type {
|
||||
ColumnUint16 meta_uint16 = 8;
|
||||
ColumnUint32 meta_uint32 = 9;
|
||||
ColumnFloat32 meta_float32 = 10;
|
||||
}
|
||||
}
|
||||
|
||||
message Snapshot {
|
||||
// All fields of the dataset, including the nested fields.
|
||||
repeated Field fields = 1;
|
||||
|
||||
repeated string data_files = 2;
|
||||
|
||||
// Snapshot version number.
|
||||
uint64 version = 3;
|
||||
|
||||
}
|
||||
|
||||
message DataFile {
|
||||
repeated int32 field_ids = 1;
|
||||
repeated RowGroup row_groups = 2;
|
||||
}
|
||||
|
||||
message RowGroup {
|
||||
int64 row_offset = 1;
|
||||
int32 row_count = 2;
|
||||
repeated ColumnSplit column_splits = 3;
|
||||
}
|
||||
|
||||
// Field metadata for a column.
|
||||
message Field {
|
||||
enum Type {
|
||||
PARENT = 0;
|
||||
REPEATED = 1;
|
||||
LEAF = 2;
|
||||
}
|
||||
Type type = 1;
|
||||
|
||||
// Fully qualified name.
|
||||
string name = 2;
|
||||
/// Field Id.
|
||||
int32 id = 3;
|
||||
/// Parent Field ID. If not set, this is a top-level column.
|
||||
int32 parent_id = 4;
|
||||
|
||||
// Logical types, support parameterized Arrow Type.
|
||||
LogicalType logical_type = 5;
|
||||
// If this field is nullable.
|
||||
bool nullable = 6;
|
||||
}
|
||||
|
||||
|
||||
message AnyValue {
|
||||
oneof value {
|
||||
bytes bytes_value = 1;
|
||||
bool bool_value = 2;
|
||||
uint64 int64_value = 3;
|
||||
uint32 int32_value = 4;
|
||||
double double_value = 5;
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user