mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-03 05:02:03 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 chrislu
|
c057ae6004 | ||
|
chrislu
|
5e42b69841 | ||
|
chrislu
|
59f44b70c3 |
@@ -44,7 +44,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@f46c48ed6d4f1227fb2d9ea62bf6bcbed315589e
|
||||
uses: actions/dependency-review-action@c090f4e553673e6e505ea70d6a95362ee12adb94
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v2
|
||||
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v2
|
||||
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
name: "helm: publish charts"
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@master
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
@@ -1,51 +0,0 @@
|
||||
name: "helm: lint and test charts"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**']
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v3
|
||||
with:
|
||||
version: v3.10.0
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
check-latest: true
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: helm/chart-testing-action@v2.4.0
|
||||
|
||||
- name: Run chart-testing (list-changed)
|
||||
id: list-changed
|
||||
run: |
|
||||
changed=$(ct list-changed --chart-dirs k8s/charts)
|
||||
if [[ -n "$changed" ]]; then
|
||||
echo "::set-output name=changed::true"
|
||||
fi
|
||||
|
||||
- name: Run chart-testing (lint)
|
||||
run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.5.0
|
||||
|
||||
- name: Run chart-testing (install)
|
||||
run: ct install --all --chart-dirs k8s/charts
|
||||
@@ -8,7 +8,7 @@
|
||||
[](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
[](https://hub.docker.com/r/chrislusf/seaweedfs/)
|
||||
[](https://search.maven.org/search?q=g:com.github.chrislusf)
|
||||
[](https://artifacthub.io/packages/search?repo=seaweedfs)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ Your support will be really appreciated by me and other supporters!
|
||||
-->
|
||||
|
||||
### Gold Sponsors
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.piknik.com)
|
||||
|
||||
---
|
||||
@@ -66,7 +66,6 @@ Table of Contents
|
||||
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
|
||||
* [Compared to GlusterFS](#compared-to-glusterfs)
|
||||
* [Compared to Ceph](#compared-to-ceph)
|
||||
* [Compared to Minio](#compared-to-minio)
|
||||
* [Dev Plan](#dev-plan)
|
||||
* [Installation Guide](#installation-guide)
|
||||
* [Disk Related Topics](#disk-related-topics)
|
||||
|
||||
@@ -3,23 +3,22 @@ module github.com/seaweedfs/seaweedfs
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.30.0
|
||||
cloud.google.com/go/storage v1.30.1
|
||||
cloud.google.com/go v0.107.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.28.0
|
||||
cloud.google.com/go/storage v1.29.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.44.244
|
||||
github.com/aws/aws-sdk-go v1.44.189
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cenkalti/backoff/v4 v4.2.1
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/eapache/go-resiliency v1.3.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
@@ -28,23 +27,23 @@ require (
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.21.0
|
||||
github.com/fclairamb/ftpserverlib v0.20.0
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-redsync/redsync/v4 v4.8.1
|
||||
github.com/go-redsync/redsync/v4 v4.7.1
|
||||
github.com/go-sql-driver/mysql v1.7.0
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/gocql/gocql v1.3.2
|
||||
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.1.2
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -52,22 +51,22 @@ require (
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
|
||||
github.com/jinzhu/copier v0.3.5
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.16.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.7
|
||||
github.com/klauspost/compress v1.15.14 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.6
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.8
|
||||
github.com/linxGnu/grocksdb v1.7.16
|
||||
github.com/lib/pq v1.10.7
|
||||
github.com/linxGnu/grocksdb v1.7.14
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.9 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -77,12 +76,12 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.1.0
|
||||
github.com/prometheus/client_golang v1.15.0
|
||||
github.com/prometheus/client_golang v1.14.0
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.9.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.0
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
@@ -90,7 +89,7 @@ require (
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.15.0
|
||||
github.com/stretchr/testify v1.8.2
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.14.4
|
||||
@@ -105,200 +104,141 @@ require (
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.7
|
||||
go.mongodb.org/mongo-driver v1.11.3
|
||||
go.mongodb.org/mongo-driver v1.11.1
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.29.0
|
||||
gocloud.dev/pubsub/natspubsub v0.29.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.29.0
|
||||
golang.org/x/crypto v0.7.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
|
||||
golang.org/x/image v0.6.0
|
||||
golang.org/x/net v0.9.0
|
||||
golang.org/x/oauth2 v0.6.0 // indirect
|
||||
golang.org/x/sys v0.7.0
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
golang.org/x/tools v0.8.0
|
||||
gocloud.dev v0.28.0
|
||||
gocloud.dev/pubsub/natspubsub v0.28.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.27.0
|
||||
golang.org/x/crypto v0.3.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.5.0
|
||||
golang.org/x/oauth2 v0.2.0 // indirect
|
||||
golang.org/x/sys v0.4.0
|
||||
golang.org/x/text v0.6.0 // indirect
|
||||
golang.org/x/tools v0.5.0
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/api v0.114.0
|
||||
google.golang.org/api v0.108.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect
|
||||
google.golang.org/grpc v1.54.0
|
||||
google.golang.org/protobuf v1.30.0
|
||||
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
|
||||
google.golang.org/grpc v1.52.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/cc/v3 v3.40.0 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||
modernc.org/libc v1.22.3 // indirect
|
||||
modernc.org/libc v1.22.2 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/memory v1.5.0 // indirect
|
||||
modernc.org/memory v1.4.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/sqlite v1.21.1
|
||||
modernc.org/sqlite v1.20.1
|
||||
modernc.org/strutil v1.1.3
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Jille/raft-grpc-transport v1.4.0
|
||||
github.com/arangodb/go-driver v1.5.2
|
||||
github.com/arangodb/go-driver v1.4.1
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hanwen/go-fuse/v2 v2.2.0
|
||||
github.com/hashicorp/raft v1.4.0
|
||||
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
|
||||
github.com/hashicorp/raft v1.3.11
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
||||
github.com/rabbitmq/amqp091-go v1.8.0
|
||||
github.com/rclone/rclone v1.62.2
|
||||
github.com/schollz/progressbar/v3 v3.13.1
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/rabbitmq/amqp091-go v1.5.0
|
||||
github.com/schollz/progressbar/v3 v3.13.0
|
||||
github.com/tikv/client-go/v2 v2.0.5
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.43.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.6
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.19.0 // indirect
|
||||
cloud.google.com/go/compute v1.14.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v0.13.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||
github.com/Unknwon/goconfig v1.0.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
cloud.google.com/go/iam v0.8.0 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/benbjohnson/clock v1.3.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
|
||||
github.com/aws/smithy-go v1.13.4 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/elastic/gosigar v0.14.2 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/fclairamb/go-log v0.4.1 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/geoffgarside/ber v1.1.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.0.8 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/gofrs/flock v0.8.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.2.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/hirochachacha/go-smb2 v1.1.0 // indirect
|
||||
github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jlaffaye/ftp v0.1.1-0.20230214004652-d84bf4be2b6e // indirect
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.7.0 // indirect
|
||||
github.com/nats-io/nats.go v1.23.0 // indirect
|
||||
github.com/montanaflynn/stats v0.6.6 // indirect
|
||||
github.com/nats-io/nats.go v1.20.0 // indirect
|
||||
github.com/nats-io/nkeys v0.3.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.1 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.32.0 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 // indirect
|
||||
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 // indirect
|
||||
github.com/pkg/xattr v0.4.9 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.23.2 // indirect
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/sony/gobreaker v0.5.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
|
||||
github.com/rivo/uniseg v0.4.3 // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stathat/consistent v1.0.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
github.com/tklauser/numcpus v0.6.0 // indirect
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
github.com/zeebo/blake3 v0.2.3 // indirect
|
||||
github.com/zeebo/errs v1.3.0 // indirect
|
||||
go.etcd.io/bbolt v1.3.7 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.7 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/mod v0.10.0 // indirect
|
||||
golang.org/x/mod v0.7.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/term v0.7.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/term v0.4.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/uint128 v1.2.0 // indirect
|
||||
storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
|
||||
storj.io/drpc v0.0.32 // indirect
|
||||
storj.io/uplink v1.10.0 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
# Artifact Hub repository metadata file
|
||||
#
|
||||
# Some settings like the verified publisher flag or the ignored packages won't
|
||||
# be applied until the next time the repository is processed. Please keep in
|
||||
# mind that the repository won't be processed if it has not changed since the
|
||||
# last time it was processed. Depending on the repository kind, this is checked
|
||||
# in a different way. For Helm http based repositories, we consider it has
|
||||
# changed if the `index.yaml` file changes. For git based repositories, it does
|
||||
# when the hash of the last commit in the branch you set up changes. This does
|
||||
# NOT apply to ownership claim operations, which are processed immediately.
|
||||
#
|
||||
|
||||
repositoryID: 5b2f1fe2-20e5-486e-9746-183484642aa2
|
||||
# owners: # (optional, used to claim repository ownership)
|
||||
# - name: username
|
||||
# email: email
|
||||
@@ -1,81 +0,0 @@
|
||||
# SEAWEEDFS - helm chart (2.x+)
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Add the helm repo
|
||||
|
||||
`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
|
||||
|
||||
### Install the helm chart
|
||||
|
||||
`helm install seaweedfs seaweedfs/seaweedfs`
|
||||
|
||||
### (Recommended) Provide `values.yaml`
|
||||
|
||||
`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
|
||||
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) and backup/HA memsql can provide.
|
||||
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer with ENV.
|
||||
* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
|
||||
|
||||
## Prerequisites
|
||||
### Database
|
||||
|
||||
leveldb is the default database this only supports one filer replica.
|
||||
|
||||
To have multiple filers a external datastore is recommened.
|
||||
|
||||
Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
```
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
|
||||
|
||||
Here is an example:
|
||||
* s3/filer/master needs the label **sw-backend=true**
|
||||
* volume need the label **sw-volume=true**
|
||||
|
||||
to label a node to be able to run all pod types in k8s:
|
||||
```
|
||||
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
|
||||
```
|
||||
|
||||
on production k8s deployment you will want each pod to have a different host,
|
||||
especially the volume server and the masters, all pods (master/volume/filer)
|
||||
should have anti-affinity rules to disallow running multiple component pods on the same host.
|
||||
|
||||
If you still want to run multiple pods of the same component (master/volume/filer) on the same host please set/update the corresponding affinity rule in values.yaml to an empty one:
|
||||
|
||||
```affinity: ""```
|
||||
|
||||
## PVC - storage class ###
|
||||
|
||||
On the volume stateful set added support for k8s PVC, currently example
|
||||
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
|
||||
https://github.com/rancher/local-path-provisioner
|
||||
|
||||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
## current instances config (AIO):
|
||||
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
|
||||
You can update the replicas count for each node type in values.yaml,
|
||||
need to add more nodes with the corresponding labels if applicable.
|
||||
|
||||
Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
|
||||
@@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.47"
|
||||
version: "3.47"
|
||||
appVersion: "3.42"
|
||||
version: "3.42"
|
||||
@@ -0,0 +1,62 @@
|
||||
# SEAWEEDFS - helm chart (2.x)
|
||||
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
|
||||
and backup/HA memsql can provide.
|
||||
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
|
||||
with ENV.
|
||||
* cert config exists and can be enabled, but not been tested.
|
||||
|
||||
## Prerequisites
|
||||
### Database
|
||||
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
```
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes node have labels which help to define which node(Host) will run which pod:
|
||||
* s3/filer/master needs the label **sw-backend=true**
|
||||
* volume need the label **sw-volume=true**
|
||||
|
||||
to label a node to be able to run all pod types in k8s:
|
||||
```
|
||||
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
|
||||
```
|
||||
|
||||
on production k8s deployment you will want each pod to have a different host,
|
||||
especially the volume server & the masters, currently all pods (master/volume/filer)
|
||||
have anti-affinity rule to disallow running multiple pod type on the same host.
|
||||
if you still want to run multiple pods of the same type (master/volume/filer) on the same host
|
||||
please set/update the corresponding affinity rule in values.yaml to an empty one:
|
||||
|
||||
```affinity: ""```
|
||||
|
||||
## PVC - storage class ###
|
||||
|
||||
on the volume stateful set added support for K8S PVC, currently example
|
||||
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
|
||||
https://github.com/rancher/local-path-provisioner
|
||||
|
||||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
## current instances config (AIO):
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
|
||||
you can update the replicas count for each node type in values.yaml,
|
||||
need to add more nodes with the corresponding labels.
|
||||
|
||||
most of the configuration are available through values.yaml
|
||||
|
||||
-6
@@ -20,12 +20,6 @@ spec:
|
||||
port: {{ .Values.filer.grpcPort }}
|
||||
targetPort: {{ .Values.filer.grpcPort }}
|
||||
protocol: TCP
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
- name: "swfs-s3"
|
||||
port: {{ .Values.filer.s3.port }}
|
||||
targetPort: {{ .Values.filer.s3.port }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
{{- if .Values.filer.metricsPort }}
|
||||
- name: "metrics"
|
||||
port: {{ .Values.filer.metricsPort }}
|
||||
+2
-8
@@ -97,12 +97,6 @@ spec:
|
||||
value: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.secretExtraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
valueFrom: {{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
@@ -168,14 +162,14 @@ spec:
|
||||
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name: seaweedfs-filer-log-volume
|
||||
mountPath: "/logs/"
|
||||
- mountPath: /etc/sw
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
|
||||
{{- if .Values.filer.enablePVC }}
|
||||
- name: data-filer
|
||||
mountPath: /data
|
||||
{{- end }}
|
||||
+2
-2
@@ -133,8 +133,8 @@ spec:
|
||||
{{- if .Values.master.garbageThreshold }}
|
||||
-garbageThreshold={{ .Values.master.garbageThreshold }} \
|
||||
{{- end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
|
||||
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
|
||||
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name : data-{{ .Release.Namespace }}
|
||||
mountPath: /data
|
||||
+1
-1
@@ -105,7 +105,7 @@ spec:
|
||||
{{- if .Values.s3.auditLogConfig }}
|
||||
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
-filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
|
||||
-filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: "/logs/"
|
||||
+3
-3
@@ -138,9 +138,9 @@ spec:
|
||||
-fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \
|
||||
{{- end }}
|
||||
-minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
|
||||
-compactionMBps={{ .Values.volume.compactionMBps }} \
|
||||
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: "{{ .Values.volume.dir }}/"
|
||||
@@ -187,7 +187,7 @@ spec:
|
||||
port: {{ .Values.volume.port }}
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 15
|
||||
periodSeconds: 90
|
||||
successThreshold: 1
|
||||
failureThreshold: 100
|
||||
timeoutSeconds: 30
|
||||
@@ -5,7 +5,7 @@ global:
|
||||
repository: ""
|
||||
imageName: chrislusf/seaweedfs
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: ""
|
||||
imagePullSecrets: imagepullsecret
|
||||
restartPolicy: Always
|
||||
loggingLevel: 1
|
||||
enableSecurity: false
|
||||
@@ -22,8 +22,8 @@ global:
|
||||
replicationPlacment: "001"
|
||||
extraEnvironmentVars:
|
||||
WEED_CLUSTER_DEFAULT: "sw"
|
||||
WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
|
||||
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
|
||||
WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333"
|
||||
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888"
|
||||
|
||||
image:
|
||||
registry: ""
|
||||
@@ -44,11 +44,11 @@ master:
|
||||
volumePreallocate: false
|
||||
volumeSizeLimitMB: 1000
|
||||
loggingOverrideLevel: null
|
||||
# number of seconds between heartbeats, default 5
|
||||
#number of seconds between heartbeats, default 5
|
||||
pulseSeconds: null
|
||||
# threshold to vacuum and reclaim spaces, default 0.3 (30%)
|
||||
#threshold to vacuum and reclaim spaces, default 0.3 (30%)
|
||||
garbageThreshold: null
|
||||
# Prometheus push interval in seconds, default 15
|
||||
#Prometheus push interval in seconds, default 15
|
||||
metricsIntervalSec: 15
|
||||
# replication type is XYZ:
|
||||
# X number of replica in other data centers
|
||||
@@ -112,10 +112,10 @@ master:
|
||||
# nodeSelector labels for master pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
nodeSelector: |
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
|
||||
# used to assign priority to master pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -241,7 +241,7 @@ volume:
|
||||
# should map directly to the value of the resources field for a PodSpec,
|
||||
# formatted as a multi-line string. By default no direct resource request
|
||||
# is made.
|
||||
resources: {}
|
||||
resources: null
|
||||
|
||||
# Toleration Settings for server pods
|
||||
# This should be a multi-line string matching the Toleration array
|
||||
@@ -251,10 +251,10 @@ volume:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
nodeSelector: |
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-volume: "true"
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-volume: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -358,10 +358,10 @@ filer:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
nodeSelector: |
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -389,7 +389,7 @@ filer:
|
||||
|
||||
# extraEnvVars is a list of extra enviroment variables to set with the stateful set.
|
||||
extraEnvironmentVars:
|
||||
WEED_MYSQL_ENABLED: "false"
|
||||
WEED_MYSQL_ENABLED: "true"
|
||||
WEED_MYSQL_HOSTNAME: "mysql-db-host"
|
||||
WEED_MYSQL_PORT: "3306"
|
||||
WEED_MYSQL_DATABASE: "sw_database"
|
||||
@@ -400,29 +400,17 @@ filer:
|
||||
# enable usage of memsql as filer backend
|
||||
WEED_MYSQL_INTERPOLATEPARAMS: "true"
|
||||
# if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
|
||||
WEED_LEVELDB2_ENABLED: "true"
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
# with http DELETE, by default the filer would check whether a folder is empty.
|
||||
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
|
||||
WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false"
|
||||
# directories under this folder will be automatically creating a separate bucket
|
||||
WEED_FILER_BUCKETS_FOLDER: "/buckets"
|
||||
|
||||
|
||||
# secret env variables
|
||||
secretExtraEnvironmentVars: []
|
||||
# WEED_POSTGRES_USERNAME:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: username
|
||||
# WEED_POSTGRES_PASSWORD:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: password
|
||||
|
||||
s3:
|
||||
enabled: true
|
||||
port: 8333
|
||||
# allow empty folders
|
||||
#allow empty folders
|
||||
allowEmptyFolder: false
|
||||
# Suffix of the host name, {bucket}.{domainName}
|
||||
domainName: ""
|
||||
@@ -442,7 +430,7 @@ s3:
|
||||
port: 8333
|
||||
metricsPort: 9327
|
||||
loggingOverrideLevel: null
|
||||
# allow empty folders
|
||||
#allow empty folders
|
||||
allowEmptyFolder: true
|
||||
# enable user & permission to s3 (need to inject to all services)
|
||||
enableAuth: false
|
||||
@@ -461,7 +449,7 @@ s3:
|
||||
# should map directly to the value of the resources field for a PodSpec,
|
||||
# formatted as a multi-line string. By default no direct resource request
|
||||
# is made.
|
||||
resources: {}
|
||||
resources: null
|
||||
|
||||
# Toleration Settings for server pods
|
||||
# This should be a multi-line string matching the Toleration array
|
||||
@@ -471,10 +459,10 @@ s3:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
nodeSelector: |
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -491,5 +479,5 @@ certificates:
|
||||
ipAddresses: []
|
||||
keyAlgorithm: rsa
|
||||
keySize: 2048
|
||||
duration: 2160h # 90d
|
||||
renewBefore: 360h # 15d
|
||||
duration: 2160h # 90d
|
||||
renewBefore: 360h # 15d
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 KiB |
@@ -5,10 +5,6 @@
|
||||
<artifactId>copier</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<properties>
|
||||
<maven.compiler.source>18</maven.compiler.source>
|
||||
<maven.compiler.target>18</maven.compiler.target>
|
||||
</properties>
|
||||
<name>copier</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
|
||||
@@ -10,8 +10,6 @@ import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.regions.Regions;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.CreateBucketRequest;
|
||||
import com.amazonaws.services.s3.model.GetBucketLocationRequest;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
|
||||
import com.amazonaws.services.s3.transfer.Upload;
|
||||
@@ -44,19 +42,6 @@ public class HighLevelMultipartUpload {
|
||||
.withS3Client(s3Client)
|
||||
.build();
|
||||
|
||||
|
||||
if (!s3Client.doesBucketExistV2(bucketName)) {
|
||||
// Because the CreateBucketRequest object doesn't specify a region, the
|
||||
// bucket is created in the region specified in the client.
|
||||
s3Client.createBucket(new CreateBucketRequest(bucketName));
|
||||
|
||||
// Verify that the bucket was created by retrieving it and checking its location.
|
||||
String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
|
||||
System.out.println("Bucket location: " + bucketLocation);
|
||||
} else {
|
||||
System.out.println("Bucket already exists");
|
||||
}
|
||||
|
||||
// TransferManager processes all transfers asynchronously,
|
||||
// so this call returns immediately.
|
||||
Upload upload = tm.upload(bucketName, keyName, file);
|
||||
|
||||
@@ -79,19 +79,7 @@ func startGenerateMetadata() {
|
||||
|
||||
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "tail",
|
||||
ClientId: 0,
|
||||
ClientEpoch: 0,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *dir,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: 0,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), metadataFollowOption, eachEntryFunc)
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
|
||||
|
||||
if tailErr != nil {
|
||||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
|
||||
|
||||
@@ -60,7 +60,6 @@ type FilerOptions struct {
|
||||
localSocket *string
|
||||
showUIDirectoryDelete *bool
|
||||
downloadMaxMBps *int
|
||||
diskType *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -90,7 +89,6 @@ func init() {
|
||||
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
|
||||
f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
|
||||
// start s3 on filer
|
||||
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
@@ -185,11 +183,6 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
|
||||
if *filerStartWebDav {
|
||||
filerWebDavOptions.filer = &filerAddress
|
||||
|
||||
if *filerWebDavOptions.disk == "" {
|
||||
filerWebDavOptions.disk = f.diskType
|
||||
}
|
||||
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerWebDavOptions.startWebDav()
|
||||
@@ -250,7 +243,6 @@ func (fo *FilerOptions) startFiler() {
|
||||
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
|
||||
ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
|
||||
DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
|
||||
DiskType: *fo.diskType,
|
||||
})
|
||||
if nfs_err != nil {
|
||||
glog.Fatalf("Filer startup error: %v", nfs_err)
|
||||
|
||||
@@ -138,19 +138,6 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
|
||||
}()
|
||||
}
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "backup_" + dataSink.GetName(),
|
||||
ClientId: clientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: sourcePath,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: startFrom.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, clientEpoch, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
}
|
||||
|
||||
@@ -196,21 +196,8 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
|
||||
})
|
||||
|
||||
metaBackup.clientEpoch++
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "meta_backup",
|
||||
ClientId: metaBackup.clientId,
|
||||
ClientEpoch: metaBackup.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *metaBackup.filerDirectory,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: startTime.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId, metaBackup.clientEpoch,
|
||||
*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -107,28 +107,16 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
|
||||
untilTsNs = time.Now().Add(-*tailStop).UnixNano()
|
||||
}
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "tail",
|
||||
ClientId: clientId,
|
||||
ClientEpoch: 0,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *tailTarget,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: time.Now().Add(-*tailStart).UnixNano(),
|
||||
StopTsNs: untilTsNs,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if !shouldPrint(resp) {
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId, 0, *tailTarget, nil,
|
||||
time.Now().Add(-*tailStart).UnixNano(), untilTsNs, 0, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if !shouldPrint(resp) {
|
||||
return nil
|
||||
}
|
||||
if err := eachEntryFunc(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := eachEntryFunc(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}, pb.TrivialOnError)
|
||||
|
||||
if tailErr != nil {
|
||||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
|
||||
|
||||
@@ -39,21 +39,8 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
|
||||
lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)
|
||||
|
||||
option.clientEpoch++
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "filer.remote.sync",
|
||||
ClientId: option.clientId,
|
||||
ClientEpoch: option.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: option.bucketsDir,
|
||||
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: lastOffsetTs.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
|
||||
option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
}
|
||||
|
||||
func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
|
||||
|
||||
@@ -63,21 +63,8 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
|
||||
lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)
|
||||
|
||||
option.clientEpoch++
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "filer.remote.sync",
|
||||
ClientId: option.clientId,
|
||||
ClientEpoch: option.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: mountedDir,
|
||||
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: lastOffsetTs.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
|
||||
mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
}
|
||||
|
||||
func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
|
||||
|
||||
@@ -287,20 +287,8 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
|
||||
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, processor.processedTsWatermark)
|
||||
})
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: clientName,
|
||||
ClientId: clientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: targetFilerSignature,
|
||||
PathPrefix: sourcePath,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: sourceFilerOffsetTsNs,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.RetryForeverOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, clientName, clientId, clientEpoch,
|
||||
sourcePath, nil, sourceFilerOffsetTsNs, 0, targetFilerSignature, processEventFnWithOffset, pb.RetryForeverOnError)
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -2,14 +2,13 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -164,11 +163,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
RaftResumeState: *masterOption.raftResumeState,
|
||||
HeartbeatInterval: *masterOption.heartbeatInterval,
|
||||
ElectionTimeout: *masterOption.electionTimeout,
|
||||
RaftBootstrap: *masterOption.raftBootstrap,
|
||||
RaftBootstrap: *m.raftBootstrap,
|
||||
}
|
||||
var raftServer *weed_server.RaftServer
|
||||
var err error
|
||||
if *masterOption.raftHashicorp {
|
||||
if *m.raftHashicorp {
|
||||
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
|
||||
glog.Fatalf("NewHashicorpRaftServer: %s", err)
|
||||
}
|
||||
@@ -181,7 +180,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
ms.SetRaftServer(raftServer)
|
||||
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
|
||||
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
|
||||
if *masterOption.raftHashicorp {
|
||||
if *m.raftHashicorp {
|
||||
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
|
||||
}
|
||||
// starting grpc server
|
||||
@@ -192,7 +191,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
|
||||
master_pb.RegisterSeaweedServer(grpcS, ms)
|
||||
if *masterOption.raftHashicorp {
|
||||
if *m.raftHashicorp {
|
||||
raftServer.TransportManager.Register(grpcS)
|
||||
} else {
|
||||
protobuf.RegisterRaftServer(grpcS, raftServer)
|
||||
@@ -205,7 +204,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
go grpcS.Serve(grpcL)
|
||||
|
||||
timeSleep := 1500 * time.Millisecond
|
||||
if !*masterOption.raftHashicorp {
|
||||
if !*m.raftHashicorp {
|
||||
go func() {
|
||||
time.Sleep(timeSleep)
|
||||
|
||||
|
||||
@@ -182,7 +182,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
DirectMount: true,
|
||||
DirectMountFlags: 0,
|
||||
//SyncRead: false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
|
||||
EnableAcl: true,
|
||||
//EnableAcl: true,
|
||||
}
|
||||
if *option.nonempty {
|
||||
fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty")
|
||||
|
||||
@@ -108,7 +108,7 @@ connection_max_open = 100
|
||||
connection_max_lifetime_seconds = 0
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
|
||||
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
|
||||
|
||||
[postgres2]
|
||||
enabled = false
|
||||
@@ -133,7 +133,7 @@ connection_max_open = 100
|
||||
connection_max_lifetime_seconds = 0
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
|
||||
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
|
||||
|
||||
[cassandra]
|
||||
# CREATE TABLE filemeta (
|
||||
@@ -344,4 +344,4 @@ cert_path=""
|
||||
# Set the private key path
|
||||
key_path=""
|
||||
# The name list used to verify the cn name
|
||||
verify_cn=""
|
||||
verify_cn=""
|
||||
@@ -97,7 +97,6 @@ func init() {
|
||||
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
|
||||
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
|
||||
masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
|
||||
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
|
||||
@@ -116,7 +115,6 @@ func init() {
|
||||
filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
|
||||
filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
|
||||
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
|
||||
serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
|
||||
@@ -255,9 +253,6 @@ func runServer(cmd *Command, args []string) bool {
|
||||
serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
|
||||
|
||||
if *isStartingFiler {
|
||||
if *filerOptions.diskType == "" && *serverOptions.v.diskType != "" {
|
||||
filerOptions.diskType = serverOptions.v.diskType
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(1 * time.Second)
|
||||
filerOptions.startFiler()
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -214,7 +215,7 @@ func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (R
|
||||
return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
buf, err := io.ReadAll(res.Body)
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return Release{}, err
|
||||
}
|
||||
@@ -264,7 +265,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
buf, err := io.ReadAll(res.Body)
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -336,7 +337,7 @@ func extractToFile(buf []byte, filename, target string) error {
|
||||
|
||||
// Write everything to a temp file
|
||||
dir := filepath.Dir(target)
|
||||
new, err := os.CreateTemp(dir, "weed")
|
||||
new, err := ioutil.TempFile(dir, "weed")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
all: gen
|
||||
|
||||
.PHONY : gen
|
||||
|
||||
gen:
|
||||
protoc columnar.proto --go_out=./columnar_pb --go-grpc_out=./columnar_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
|
||||
@@ -0,0 +1,32 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
type ColumnUint16 struct {
|
||||
}
|
||||
|
||||
const SIZE_Uint16 = 2
|
||||
|
||||
func (c *ColumnUint16) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint16 {
|
||||
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
|
||||
return binary.BigEndian.Uint16(buf)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func WriteUint16s(buf []byte, data []uint16) (err error) {
|
||||
off := 0
|
||||
size := len(data)
|
||||
if len(buf) < size<<1 {
|
||||
return fmt.Errorf("buf too small")
|
||||
}
|
||||
for _, dat := range data {
|
||||
binary.BigEndian.PutUint16(buf[off:], dat)
|
||||
off += SIZE_Uint16
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
type ColumnUint32 struct {
|
||||
}
|
||||
|
||||
const SIZE_Uint32 = 4
|
||||
|
||||
func (c *ColumnUint32) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint32 {
|
||||
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
|
||||
return binary.BigEndian.Uint32(buf)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func WriteUint32s(buf []byte, data []uint32) (err error) {
|
||||
off := 0
|
||||
size := len(data)
|
||||
if len(buf) < size<<2 {
|
||||
return fmt.Errorf("buf too small")
|
||||
}
|
||||
for _, dat := range data {
|
||||
binary.BigEndian.PutUint32(buf[off:], dat)
|
||||
off += SIZE_Uint32
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
type ColumnUint64 struct {
|
||||
}
|
||||
|
||||
const SIZE_Uint64 = 8
|
||||
|
||||
func (c *ColumnUint64) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint64 {
|
||||
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint64); n == SIZE_Uint64 && err == nil {
|
||||
return binary.BigEndian.Uint64(buf)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func WriteUint64s(buf []byte, data []uint64) (err error) {
|
||||
off := 0
|
||||
size := len(data)
|
||||
if len(buf) < size<<3 {
|
||||
return fmt.Errorf("buf too small")
|
||||
}
|
||||
for _, dat := range data {
|
||||
binary.BigEndian.PutUint64(buf[off:], dat)
|
||||
off += SIZE_Uint64
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package columnar_pb;
|
||||
|
||||
option go_package = "github.com/seaweedfs/seaweedfs/weed/data/columnar_pb";
|
||||
|
||||
message FileId {
|
||||
uint32 volume_id = 1;
|
||||
uint64 file_key = 2;
|
||||
fixed32 cookie = 3;
|
||||
}
|
||||
|
||||
enum LogicalType {
|
||||
Uint8 = 0;
|
||||
Uint16 = 1;
|
||||
Float32 = 4;
|
||||
}
|
||||
|
||||
message ColumnUint16 {
|
||||
uint32 base = 1;
|
||||
uint32 min = 3;
|
||||
uint32 max = 4;
|
||||
}
|
||||
|
||||
message ColumnUint32 {
|
||||
uint32 base = 1;
|
||||
uint32 min = 3;
|
||||
uint32 max = 4;
|
||||
}
|
||||
|
||||
message ColumnFloat32 {
|
||||
uint32 min = 3;
|
||||
uint32 max = 4;
|
||||
}
|
||||
|
||||
message ColumnSplit {
|
||||
// The ids of the fields/columns in this file
|
||||
int32 field_id = 1;
|
||||
FileId file_id = 2;
|
||||
int64 row_offset = 3;
|
||||
int32 row_count = 4;
|
||||
|
||||
oneof storage_type {
|
||||
ColumnUint16 meta_uint16 = 8;
|
||||
ColumnUint32 meta_uint32 = 9;
|
||||
ColumnFloat32 meta_float32 = 10;
|
||||
}
|
||||
}
|
||||
|
||||
message Snapshot {
|
||||
// All fields of the dataset, including the nested fields.
|
||||
repeated Field fields = 1;
|
||||
|
||||
repeated string data_files = 2;
|
||||
|
||||
// Snapshot version number.
|
||||
uint64 version = 3;
|
||||
|
||||
}
|
||||
|
||||
message DataFile {
|
||||
repeated int32 field_ids = 1;
|
||||
repeated RowGroup row_groups = 2;
|
||||
}
|
||||
|
||||
message RowGroup {
|
||||
int64 row_offset = 1;
|
||||
int32 row_count = 2;
|
||||
repeated ColumnSplit column_splits = 3;
|
||||
}
|
||||
|
||||
// Field metadata for a column.
|
||||
message Field {
|
||||
enum Type {
|
||||
PARENT = 0;
|
||||
REPEATED = 1;
|
||||
LEAF = 2;
|
||||
}
|
||||
Type type = 1;
|
||||
|
||||
// Fully qualified name.
|
||||
string name = 2;
|
||||
/// Field Id.
|
||||
int32 id = 3;
|
||||
/// Parent Field ID. If not set, this is a top-level column.
|
||||
int32 parent_id = 4;
|
||||
|
||||
// Logical types, support parameterized Arrow Type.
|
||||
LogicalType logical_type = 5;
|
||||
// If this field is nullable.
|
||||
bool nullable = 6;
|
||||
}
|
||||
|
||||
|
||||
message AnyValue {
|
||||
oneof value {
|
||||
bytes bytes_value = 1;
|
||||
bool bool_value = 2;
|
||||
uint64 int64_value = 3;
|
||||
uint32 int32_value = 4;
|
||||
double double_value = 5;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,91 @@
|
||||
package data
|
||||
|
||||
import "fmt"
|
||||
|
||||
type Datum interface {
|
||||
Compare(other Datum) (int, error)
|
||||
}
|
||||
type Datums []Datum
|
||||
|
||||
type DUint16 uint16
|
||||
type DUint32 uint32
|
||||
type DUint64 uint64
|
||||
type dNull struct{}
|
||||
|
||||
var (
|
||||
DNull Datum = dNull{}
|
||||
)
|
||||
|
||||
func (d dNull) Compare(other Datum) (int, error) {
|
||||
if other == DNull {
|
||||
return 0, nil
|
||||
}
|
||||
return -1, nil
|
||||
}
|
||||
|
||||
func NewDUint16(d DUint16) *DUint16 {
|
||||
return &d
|
||||
}
|
||||
func NewDUint32(d DUint32) *DUint32 {
|
||||
return &d
|
||||
}
|
||||
func NewDUint64(d DUint64) *DUint64 {
|
||||
return &d
|
||||
}
|
||||
|
||||
func (d *DUint16) Compare(other Datum) (int, error) {
|
||||
if other == DNull {
|
||||
return 1, nil
|
||||
}
|
||||
thisV := *d
|
||||
var otherV DUint16
|
||||
switch t := other.(type) {
|
||||
case *DUint16:
|
||||
otherV = *t
|
||||
default:
|
||||
return 0, fmt.Errorf("unsupported")
|
||||
}
|
||||
if thisV < otherV {
|
||||
return -1, nil
|
||||
}
|
||||
if thisV > otherV {
|
||||
return 1, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
func (d *DUint32) Compare(other Datum) (int, error) {
|
||||
if other == DNull {
|
||||
return 1, nil
|
||||
}
|
||||
thisV := *d
|
||||
var otherV DUint32
|
||||
switch t := other.(type) {
|
||||
case *DUint32:
|
||||
otherV = *t
|
||||
}
|
||||
if thisV < otherV {
|
||||
return -1, nil
|
||||
}
|
||||
if thisV > otherV {
|
||||
return 1, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
func (d *DUint64) Compare(other Datum) (int, error) {
|
||||
if other == DNull {
|
||||
return 1, nil
|
||||
}
|
||||
thisV := *d
|
||||
var otherV DUint64
|
||||
switch t := other.(type) {
|
||||
case *DUint64:
|
||||
otherV = *t
|
||||
}
|
||||
if thisV < otherV {
|
||||
return -1, nil
|
||||
}
|
||||
if thisV > otherV {
|
||||
return 1, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRead(t *testing.T) {
|
||||
x := make([]uint16, 128)
|
||||
y := make([]uint32, 128)
|
||||
z := make([]uint64, 128)
|
||||
|
||||
for i := range x {
|
||||
x[i] = uint16(i)
|
||||
}
|
||||
for i := range y {
|
||||
y[i] = uint32(i * 2)
|
||||
}
|
||||
for i := range z {
|
||||
z[i] = uint64(i * 3)
|
||||
}
|
||||
|
||||
xbuf := make([]byte, len(x)*SIZE_Uint16)
|
||||
ybuf := make([]byte, len(x)*SIZE_Uint32)
|
||||
zbuf := make([]byte, len(x)*SIZE_Uint64)
|
||||
|
||||
WriteUint16s(xbuf, x)
|
||||
WriteUint32s(ybuf, y)
|
||||
WriteUint64s(zbuf, z)
|
||||
|
||||
df := &DataFile{
|
||||
xbuf: xbuf,
|
||||
ybuf: ybuf,
|
||||
zbuf: zbuf,
|
||||
xLen: len(xbuf),
|
||||
yLen: len(ybuf),
|
||||
zLen: len(zbuf),
|
||||
xReaderAt: util.NewBytesReader(xbuf),
|
||||
yReaderAt: util.NewBytesReader(ybuf),
|
||||
zReaderAt: util.NewBytesReader(zbuf),
|
||||
}
|
||||
|
||||
dataLayout := make(map[FieldName]DataLayout)
|
||||
dataLayout["x"] = DataLayout{
|
||||
LayoutType: Uint16,
|
||||
SortType: Unsorted,
|
||||
}
|
||||
dataLayout["y"] = DataLayout{
|
||||
LayoutType: Uint32,
|
||||
SortType: Unsorted,
|
||||
}
|
||||
dataLayout["z"] = DataLayout{
|
||||
LayoutType: Uint64,
|
||||
SortType: Unsorted,
|
||||
}
|
||||
|
||||
rows, err := df.ReadRows("x", dataLayout, Equal, NewDUint16(65))
|
||||
if err != nil {
|
||||
fmt.Printf("err: %v", err)
|
||||
return
|
||||
}
|
||||
for _, row := range rows {
|
||||
fmt.Printf("row %d width %d ", row.index, len(row.Datums))
|
||||
for i, d := range row.Datums {
|
||||
fmt.Printf("%d: %v ", i, d)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type Operator int32
|
||||
type LayoutType int32
|
||||
type SortType int32
|
||||
|
||||
const (
|
||||
Equal Operator = 0
|
||||
GreaterThan
|
||||
GreaterOrEqual
|
||||
LessThan
|
||||
LessOrEqual
|
||||
|
||||
Uint16 LayoutType = 0
|
||||
Uint32 = 1
|
||||
Uint64 = 2
|
||||
|
||||
Unsorted SortType = 0
|
||||
Ascending
|
||||
Descending
|
||||
)
|
||||
|
||||
type DataFile struct {
|
||||
xbuf []byte
|
||||
ybuf []byte
|
||||
zbuf []byte
|
||||
xReaderAt io.ReaderAt
|
||||
yReaderAt io.ReaderAt
|
||||
zReaderAt io.ReaderAt
|
||||
xLen int
|
||||
yLen int
|
||||
zLen int
|
||||
}
|
||||
|
||||
type DataLayout struct {
|
||||
LayoutType
|
||||
SortType
|
||||
}
|
||||
|
||||
type FieldName string
|
||||
|
||||
func (d *DataFile) ReadRows(field FieldName, layout map[FieldName]DataLayout, op Operator, operand Datum) (rows []*Row, err error) {
|
||||
if field == "x" {
|
||||
rows, err = pushDownReadRows(d.xReaderAt, d.xLen, layout[field], op, operand)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = hydrateRows(d.yReaderAt, d.yLen, layout["y"], rows)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = hydrateRows(d.zReaderAt, d.zLen, layout["z"], rows)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if field == "y" {
|
||||
rows, err = pushDownReadRows(d.yReaderAt, d.yLen, layout[field], op, operand)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = hydrateRows(d.xReaderAt, d.xLen, layout["x"], rows)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = hydrateRows(d.zReaderAt, d.zLen, layout["z"], rows)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type Row struct {
|
||||
index int
|
||||
Datums
|
||||
}
|
||||
|
||||
func pushDownReadRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, op Operator, operand Datum) (rows []*Row, err error) {
|
||||
if layout.LayoutType == Uint16 {
|
||||
if layout.SortType == Unsorted {
|
||||
buf := make([]byte, SIZE_Uint16)
|
||||
for i := 0; i < dataLen; i += SIZE_Uint16 {
|
||||
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint16 && err == nil {
|
||||
d := NewDUint16(DUint16(binary.BigEndian.Uint16(buf)))
|
||||
cmp, err := d.Compare(operand)
|
||||
if err != nil {
|
||||
return rows, err
|
||||
}
|
||||
if cmp == 0 && op == Equal {
|
||||
println(1)
|
||||
rows = append(rows, &Row{
|
||||
index: i / SIZE_Uint16,
|
||||
Datums: []Datum{d},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if layout.LayoutType == Uint32 {
|
||||
if layout.SortType == Unsorted {
|
||||
buf := make([]byte, SIZE_Uint32)
|
||||
for i := 0; i < dataLen; i += SIZE_Uint32 {
|
||||
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint32 && err == nil {
|
||||
d := NewDUint32(DUint32(binary.BigEndian.Uint32(buf)))
|
||||
cmp, err := d.Compare(operand)
|
||||
if err != nil {
|
||||
return rows, err
|
||||
}
|
||||
if cmp == 0 && op == Equal {
|
||||
println(2)
|
||||
rows = append(rows, &Row{
|
||||
index: i / SIZE_Uint32,
|
||||
Datums: []Datum{d},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if layout.LayoutType == Uint64 {
|
||||
if layout.SortType == Unsorted {
|
||||
buf := make([]byte, SIZE_Uint64)
|
||||
for i := 0; i < dataLen; i += SIZE_Uint64 {
|
||||
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint64 && err == nil {
|
||||
d := NewDUint64(DUint64(binary.BigEndian.Uint64(buf)))
|
||||
cmp, err := d.Compare(operand)
|
||||
if err != nil {
|
||||
return rows, err
|
||||
}
|
||||
if cmp == 0 && op == Equal {
|
||||
println(2)
|
||||
rows = append(rows, &Row{
|
||||
index: i / SIZE_Uint64,
|
||||
Datums: []Datum{d},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func hydrateRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, rows []*Row) (err error) {
|
||||
if layout.LayoutType == Uint16 {
|
||||
if layout.SortType == Unsorted {
|
||||
buf := make([]byte, SIZE_Uint16)
|
||||
for _, row := range rows {
|
||||
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
|
||||
t := binary.BigEndian.Uint16(buf)
|
||||
d := NewDUint16(DUint16(t))
|
||||
println(3, "add", t)
|
||||
row.Datums = append(row.Datums, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if layout.LayoutType == Uint32 {
|
||||
if layout.SortType == Unsorted {
|
||||
buf := make([]byte, SIZE_Uint32)
|
||||
for _, row := range rows {
|
||||
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
|
||||
t := binary.BigEndian.Uint32(buf)
|
||||
d := NewDUint32(DUint32(t))
|
||||
println(4, "add", t)
|
||||
row.Datums = append(row.Datums, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if layout.LayoutType == Uint64 {
|
||||
if layout.SortType == Unsorted {
|
||||
buf := make([]byte, SIZE_Uint64)
|
||||
for _, row := range rows {
|
||||
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint64); n == SIZE_Uint64 && err == nil {
|
||||
t := binary.BigEndian.Uint64(buf)
|
||||
d := NewDUint64(DUint64(t))
|
||||
println(5, "add", t)
|
||||
row.Datums = append(row.Datums, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -13,8 +13,7 @@ type FileChunkSection struct {
|
||||
visibleIntervals *IntervalList[*VisibleInterval]
|
||||
chunkViews *IntervalList[*ChunkView]
|
||||
reader *ChunkReadAt
|
||||
lock sync.RWMutex
|
||||
isPrepared bool
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
func NewFileChunkSection(si SectionIndex) *FileChunkSection {
|
||||
@@ -62,19 +61,6 @@ func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]st
|
||||
}
|
||||
|
||||
func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64) {
|
||||
if section.isPrepared {
|
||||
section.reader.fileSize = fileSize
|
||||
return
|
||||
}
|
||||
|
||||
section.lock.Lock()
|
||||
defer section.lock.Unlock()
|
||||
|
||||
if section.isPrepared {
|
||||
section.reader.fileSize = fileSize
|
||||
return
|
||||
}
|
||||
|
||||
if section.visibleIntervals == nil {
|
||||
section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
|
||||
section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks)
|
||||
@@ -90,25 +76,23 @@ func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64)
|
||||
if section.reader == nil {
|
||||
section.reader = NewChunkReaderAtFromClient(group.readerCache, section.chunkViews, min(int64(section.sectionIndex+1)*SectionSize, fileSize))
|
||||
}
|
||||
|
||||
section.isPrepared = true
|
||||
section.reader.fileSize = fileSize
|
||||
}
|
||||
|
||||
func (section *FileChunkSection) readDataAt(group *ChunkGroup, fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
|
||||
section.lock.Lock()
|
||||
defer section.lock.Unlock()
|
||||
|
||||
section.setupForRead(group, fileSize)
|
||||
section.lock.RLock()
|
||||
defer section.lock.RUnlock()
|
||||
|
||||
return section.reader.ReadAtWithTime(buff, offset)
|
||||
}
|
||||
|
||||
func (section *FileChunkSection) DataStartOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
|
||||
section.lock.Lock()
|
||||
defer section.lock.Unlock()
|
||||
|
||||
section.setupForRead(group, fileSize)
|
||||
section.lock.RLock()
|
||||
defer section.lock.RUnlock()
|
||||
|
||||
for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
|
||||
visible := x.Value
|
||||
@@ -124,10 +108,10 @@ func (section *FileChunkSection) DataStartOffset(group *ChunkGroup, offset int64
|
||||
}
|
||||
|
||||
func (section *FileChunkSection) NextStopOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
|
||||
section.lock.Lock()
|
||||
defer section.lock.Unlock()
|
||||
|
||||
section.setupForRead(group, fileSize)
|
||||
section.lock.RLock()
|
||||
defer section.lock.RUnlock()
|
||||
|
||||
isAfterOffset := false
|
||||
for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
|
||||
|
||||
+4
-17
@@ -87,23 +87,10 @@ func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []*
|
||||
|
||||
glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId)
|
||||
f.UniqueFilerEpoch++
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "bootstrap",
|
||||
ClientId: f.UniqueFilerId,
|
||||
ClientEpoch: f.UniqueFilerEpoch,
|
||||
SelfSignature: f.Signature,
|
||||
PathPrefix: "/",
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: snapshotTime.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.FatalOnError,
|
||||
}
|
||||
|
||||
err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return Replay(f.Store, resp)
|
||||
})
|
||||
err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, "bootstrap", f.UniqueFilerId, f.UniqueFilerEpoch, "/", nil,
|
||||
0, snapshotTime.UnixNano(), f.Signature, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return Replay(f.Store, resp)
|
||||
}, pb.FatalOnError)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -125,8 +125,7 @@ func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
|
||||
if string(key) == locationPrefix {
|
||||
return true
|
||||
}
|
||||
key = bytes.Clone(key)
|
||||
_ = rules.Put(key, value)
|
||||
rules.Put(key, value)
|
||||
return true
|
||||
})
|
||||
fc.rules = rules
|
||||
|
||||
@@ -27,7 +27,7 @@ func (interval *Interval[T]) Size() int64 {
|
||||
type IntervalList[T IntervalValue] struct {
|
||||
head *Interval[T]
|
||||
tail *Interval[T]
|
||||
Lock sync.RWMutex
|
||||
Lock sync.Mutex
|
||||
}
|
||||
|
||||
func NewIntervalList[T IntervalValue]() *IntervalList[T] {
|
||||
@@ -248,8 +248,8 @@ func (list *IntervalList[T]) overlayInterval(interval *Interval[T]) {
|
||||
}
|
||||
|
||||
func (list *IntervalList[T]) Len() int {
|
||||
list.Lock.RLock()
|
||||
defer list.Lock.RUnlock()
|
||||
list.Lock.Lock()
|
||||
defer list.Lock.Unlock()
|
||||
|
||||
var count int
|
||||
for t := list.head; t != nil; t = t.Next {
|
||||
|
||||
@@ -106,8 +106,8 @@ func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) {
|
||||
|
||||
c.readerPattern.MonitorReadAt(offset, len(p))
|
||||
|
||||
c.chunkViews.Lock.RLock()
|
||||
defer c.chunkViews.Lock.RUnlock()
|
||||
c.chunkViews.Lock.Lock()
|
||||
defer c.chunkViews.Lock.Unlock()
|
||||
|
||||
// glog.V(4).Infof("ReadAt [%d,%d) of total file size %d bytes %d chunk views", offset, offset+int64(len(p)), c.fileSize, len(c.chunkViews))
|
||||
n, _, err = c.doReadAt(p, offset)
|
||||
@@ -118,8 +118,8 @@ func (c *ChunkReadAt) ReadAtWithTime(p []byte, offset int64) (n int, ts int64, e
|
||||
|
||||
c.readerPattern.MonitorReadAt(offset, len(p))
|
||||
|
||||
c.chunkViews.Lock.RLock()
|
||||
defer c.chunkViews.Lock.RUnlock()
|
||||
c.chunkViews.Lock.Lock()
|
||||
defer c.chunkViews.Lock.Unlock()
|
||||
|
||||
// glog.V(4).Infof("ReadAt [%d,%d) of total file size %d bytes %d chunk views", offset, offset+int64(len(p)), c.fileSize, len(c.chunkViews))
|
||||
return c.doReadAt(p, offset)
|
||||
|
||||
+22
-15
@@ -300,23 +300,30 @@ func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
|
||||
if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
|
||||
return nil
|
||||
}
|
||||
// glog.V(2).Infof("c.chunkView: %v buffer:[%d,%d) offset:%d totalSize:%d", c.chunkView, c.bufferOffset, c.bufferOffset+int64(len(c.buffer)), offset, c.totalSize)
|
||||
if c.chunkView == nil {
|
||||
return io.EOF
|
||||
}
|
||||
|
||||
// find a possible chunk view
|
||||
p := c.chunkView
|
||||
for p != nil {
|
||||
chunk := p.Value
|
||||
// glog.V(2).Infof("prepareBufferFor check chunk:[%d,%d)", chunk.ViewOffset, chunk.ViewOffset+int64(chunk.ViewSize))
|
||||
if insideChunk(offset, chunk) {
|
||||
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
|
||||
c.chunkView = p
|
||||
return c.fetchChunkToBuffer(chunk)
|
||||
}
|
||||
// fmt.Printf("fetch for offset %d\n", offset)
|
||||
c.chunkView = c.chunkView.Next
|
||||
if c.chunkView == nil {
|
||||
return io.EOF
|
||||
}
|
||||
|
||||
// positioning within the new chunk
|
||||
chunk := c.chunkView.Value
|
||||
if insideChunk(offset, chunk) {
|
||||
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
|
||||
return c.fetchChunkToBuffer(chunk)
|
||||
}
|
||||
if offset < c.bufferOffset {
|
||||
p = p.Prev
|
||||
} else {
|
||||
p = p.Next
|
||||
} else {
|
||||
for p := c.head; p != nil; p = p.Next {
|
||||
chunk = p.Value
|
||||
if insideChunk(offset, chunk) {
|
||||
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
|
||||
return c.fetchChunkToBuffer(chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+47
-47
@@ -1,47 +1,47 @@
|
||||
package images
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
|
||||
srcImage, _, err := image.Decode(read)
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
return read, err
|
||||
}
|
||||
|
||||
bounds := srcImage.Bounds()
|
||||
if x2 > bounds.Dx() || y2 > bounds.Dy() {
|
||||
read.Seek(0, 0)
|
||||
return read, nil
|
||||
}
|
||||
|
||||
rectangle := image.Rect(x1, y1, x2, y2)
|
||||
dstImage := imaging.Crop(srcImage, rectangle)
|
||||
var buf bytes.Buffer
|
||||
switch ext {
|
||||
case ".jpg", ".jpeg":
|
||||
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
case ".png":
|
||||
if err = png.Encode(&buf, dstImage); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
case ".gif":
|
||||
if err = gif.Encode(&buf, dstImage, nil); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
}
|
||||
return bytes.NewReader(buf.Bytes()), err
|
||||
}
|
||||
package images
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
|
||||
srcImage, _, err := image.Decode(read)
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
return read, err
|
||||
}
|
||||
|
||||
bounds := srcImage.Bounds()
|
||||
if x2 > bounds.Dx() || y2 > bounds.Dy() {
|
||||
read.Seek(0, 0)
|
||||
return read, nil
|
||||
}
|
||||
|
||||
rectangle := image.Rect(x1, y1, x2, y2)
|
||||
dstImage := imaging.Crop(srcImage, rectangle)
|
||||
var buf bytes.Buffer
|
||||
switch ext {
|
||||
case ".jpg", ".jpeg":
|
||||
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
case ".png":
|
||||
if err = png.Encode(&buf, dstImage); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
case ".gif":
|
||||
if err = gif.Encode(&buf, dstImage, nil); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
}
|
||||
return bytes.NewReader(buf.Bytes()), err
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ type FileHandle struct {
|
||||
fh FileHandleId
|
||||
counter int64
|
||||
entry *LockedEntry
|
||||
entryLock sync.RWMutex
|
||||
entryLock sync.Mutex
|
||||
entryChunkGroup *filer.ChunkGroup
|
||||
inode uint64
|
||||
wfs *WFS
|
||||
@@ -27,7 +27,8 @@ type FileHandle struct {
|
||||
dirtyPages *PageWriter
|
||||
reader *filer.ChunkReadAt
|
||||
contentType string
|
||||
sync.RWMutex
|
||||
handle uint64
|
||||
sync.Mutex
|
||||
|
||||
isDeleted bool
|
||||
|
||||
@@ -102,9 +103,6 @@ func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
|
||||
}
|
||||
|
||||
func (fh *FileHandle) ReleaseHandle() {
|
||||
fh.Lock()
|
||||
defer fh.Unlock()
|
||||
|
||||
fh.entryLock.Lock()
|
||||
defer fh.entryLock.Unlock()
|
||||
|
||||
|
||||
@@ -23,8 +23,8 @@ func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64, tsNs in
|
||||
}
|
||||
|
||||
func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, error) {
|
||||
fh.entryLock.RLock()
|
||||
defer fh.entryLock.RUnlock()
|
||||
fh.entryLock.Lock()
|
||||
defer fh.entryLock.Unlock()
|
||||
|
||||
fileFullPath := fh.FullPath()
|
||||
|
||||
@@ -53,7 +53,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e
|
||||
return 0, 0, io.EOF
|
||||
}
|
||||
|
||||
if offset < int64(len(entry.Content)) {
|
||||
if offset+int64(len(buff)) <= int64(len(entry.Content)) {
|
||||
totalRead := copy(buff, entry.Content[offset:])
|
||||
glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
|
||||
return int64(totalRead), 0, nil
|
||||
|
||||
@@ -58,21 +58,9 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
|
||||
}
|
||||
|
||||
var clientEpoch int32
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "mount",
|
||||
ClientId: selfSignature,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: selfSignature,
|
||||
PathPrefix: dir,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: lastTsNs,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.FatalOnError,
|
||||
}
|
||||
util.RetryForever("followMetaUpdates", func() error {
|
||||
clientEpoch++
|
||||
return pb.WithFilerClientFollowMetadata(client, metadataFollowOption, processEventFn)
|
||||
return pb.WithFilerClientFollowMetadata(client, "mount", selfSignature, clientEpoch, dir, nil, &lastTsNs, 0, selfSignature, processEventFn, pb.FatalOnError)
|
||||
}, func(err error) bool {
|
||||
glog.Errorf("follow metadata updates: %v", err)
|
||||
return true
|
||||
|
||||
@@ -58,12 +58,12 @@ func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name strin
|
||||
inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.Crtime.Unix(), localEntry.IsDirectory(), len(localEntry.HardLinkId) > 0, localEntry.Inode, true)
|
||||
|
||||
if fh, found := wfs.fhmap.FindFileHandle(inode); found {
|
||||
fh.entryLock.RLock()
|
||||
fh.entryLock.Lock()
|
||||
if entry := fh.GetEntry(); entry != nil {
|
||||
glog.V(4).Infof("lookup opened file %s size %d", dirPath.Child(localEntry.Name()), filer.FileSize(entry))
|
||||
localEntry = filer.FromPbEntry(string(dirPath), entry)
|
||||
}
|
||||
fh.entryLock.RUnlock()
|
||||
fh.entryLock.Unlock()
|
||||
}
|
||||
|
||||
wfs.outputFilerEntry(out, inode, localEntry)
|
||||
|
||||
@@ -3,16 +3,14 @@ package mount
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"os"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
/** Create a directory
|
||||
@@ -56,10 +54,9 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
|
||||
defer wfs.mapPbIdFromFilerToLocal(newEntry)
|
||||
|
||||
request := &filer_pb.CreateEntryRequest{
|
||||
Directory: string(dirFullPath),
|
||||
Entry: newEntry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
SkipCheckParentDirectory: true,
|
||||
Directory: string(dirFullPath),
|
||||
Entry: newEntry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
}
|
||||
|
||||
glog.V(1).Infof("mkdir: %v", request)
|
||||
|
||||
@@ -46,14 +46,18 @@ func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn)
|
||||
// lock source and target file handles
|
||||
fhOut.Lock()
|
||||
defer fhOut.Unlock()
|
||||
fhOut.entryLock.Lock()
|
||||
defer fhOut.entryLock.Unlock()
|
||||
|
||||
if fhOut.entry == nil {
|
||||
return 0, fuse.ENOENT
|
||||
}
|
||||
|
||||
if fhIn.fh != fhOut.fh {
|
||||
fhIn.RLock()
|
||||
defer fhIn.RUnlock()
|
||||
fhIn.Lock()
|
||||
defer fhIn.Unlock()
|
||||
fhIn.entryLock.Lock()
|
||||
defer fhIn.entryLock.Unlock()
|
||||
}
|
||||
|
||||
// directories are not supported
|
||||
|
||||
@@ -35,10 +35,10 @@ func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekO
|
||||
}
|
||||
|
||||
// lock the file until the proper offset was calculated
|
||||
fh.RLock()
|
||||
defer fh.RUnlock()
|
||||
fh.entryLock.RLock()
|
||||
defer fh.entryLock.RUnlock()
|
||||
fh.Lock()
|
||||
defer fh.Unlock()
|
||||
fh.entryLock.Lock()
|
||||
defer fh.entryLock.Unlock()
|
||||
|
||||
fileSize := int64(filer.FileSize(fh.GetEntry()))
|
||||
offset := max(int64(in.Offset), 0)
|
||||
|
||||
@@ -41,8 +41,8 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse
|
||||
return nil, fuse.ENOENT
|
||||
}
|
||||
|
||||
fh.RLock()
|
||||
defer fh.RUnlock()
|
||||
fh.Lock()
|
||||
defer fh.Unlock()
|
||||
|
||||
offset := int64(in.Offset)
|
||||
totalRead, err := readDataByFileHandle(buff, fh, offset)
|
||||
|
||||
@@ -96,7 +96,7 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
|
||||
fileFullPath := fh.FullPath()
|
||||
dir, name := fileFullPath.DirAndName()
|
||||
// send the data to the OS
|
||||
glog.V(4).Infof("doFlush %s fh %d", fileFullPath, fh.fh)
|
||||
glog.V(4).Infof("doFlush %s fh %d", fileFullPath, fh.handle)
|
||||
|
||||
if !wfs.IsOverQuota {
|
||||
if err := fh.dirtyPages.FlushData(); err != nil {
|
||||
@@ -177,7 +177,7 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("%v fh %d flush: %v", fileFullPath, fh.fh, err)
|
||||
glog.Errorf("%v fh %d flush: %v", fileFullPath, fh.handle, err)
|
||||
return fuse.EIO
|
||||
}
|
||||
|
||||
|
||||
@@ -2,14 +2,12 @@ package mount
|
||||
|
||||
import (
|
||||
"context"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
/*
|
||||
@@ -74,8 +72,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
|
||||
HardLinkId: oldEntry.HardLinkId,
|
||||
HardLinkCounter: oldEntry.HardLinkCounter,
|
||||
},
|
||||
Signatures: []int32{wfs.signature},
|
||||
SkipCheckParentDirectory: true,
|
||||
Signatures: []int32{wfs.signature},
|
||||
}
|
||||
|
||||
// apply changes to the filer, and also apply to local metaCache
|
||||
|
||||
@@ -3,15 +3,13 @@ package mount
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
/** Create a symbolic link */
|
||||
@@ -44,8 +42,7 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
|
||||
SymlinkTarget: target,
|
||||
},
|
||||
},
|
||||
Signatures: []int32{wfs.signature},
|
||||
SkipCheckParentDirectory: true,
|
||||
Signatures: []int32{wfs.signature},
|
||||
}
|
||||
|
||||
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
@@ -5,6 +5,11 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
@@ -14,12 +19,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
type UploadOption struct {
|
||||
@@ -121,8 +120,7 @@ func UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.A
|
||||
return true
|
||||
})
|
||||
} else {
|
||||
uploadErrList := []string{"transport", "is read only"}
|
||||
err = util.MultiRetry("uploadWithRetry", uploadErrList, doUploadFunc)
|
||||
err = util.Retry("uploadWithRetry", doUploadFunc)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
+18
-31
@@ -19,35 +19,22 @@ const (
|
||||
RetryForeverOnError
|
||||
)
|
||||
|
||||
// MetadataFollowOption is used to control the behavior of the metadata following
|
||||
// process. Part of it is used as a cursor to resume the following process.
|
||||
type MetadataFollowOption struct {
|
||||
ClientName string
|
||||
ClientId int32
|
||||
ClientEpoch int32
|
||||
SelfSignature int32
|
||||
PathPrefix string
|
||||
AdditionalPathPrefixes []string
|
||||
DirectoriesToWatch []string
|
||||
StartTsNs int64
|
||||
StopTsNs int64
|
||||
EventErrorType EventErrorType
|
||||
}
|
||||
|
||||
type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error
|
||||
|
||||
func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {
|
||||
func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, clientName string, clientId int32, clientEpoch int32,
|
||||
pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, untilTsNs int64, selfSignature int32,
|
||||
processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) error {
|
||||
|
||||
err := WithFilerClient(true, option.SelfSignature, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(option, processEventFn))
|
||||
err := WithFilerClient(true, clientId, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(clientName, clientId, clientEpoch, pathPrefix, additionalPathPrefixes, nil, &lastTsNs, untilTsNs, selfSignature, processEventFn, eventErrorType))
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribing filer meta change: %v", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {
|
||||
func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, clientName string, clientId int32, clientEpoch int32, pathPrefix string, directoriesToWatch []string, lastTsNs *int64, untilTsNs int64, selfSignature int32, processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) error {
|
||||
|
||||
err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(option, processEventFn))
|
||||
err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(clientName, clientId, clientEpoch, pathPrefix, nil, directoriesToWatch, lastTsNs, untilTsNs, selfSignature, processEventFn, eventErrorType))
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribing filer meta change: %v", err)
|
||||
}
|
||||
@@ -55,20 +42,20 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, option *Met
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn ProcessMetadataFunc) func(client filer_pb.SeaweedFilerClient) error {
|
||||
func makeSubscribeMetadataFunc(clientName string, clientId int32, clientEpoch int32, pathPrefix string, additionalPathPrefixes []string, directoriesToWatch []string, lastTsNs *int64, untilTsNs int64, selfSignature int32, processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) func(client filer_pb.SeaweedFilerClient) error {
|
||||
return func(client filer_pb.SeaweedFilerClient) error {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
|
||||
ClientName: option.ClientName,
|
||||
PathPrefix: option.PathPrefix,
|
||||
PathPrefixes: option.AdditionalPathPrefixes,
|
||||
Directories: option.DirectoriesToWatch,
|
||||
SinceNs: option.StartTsNs,
|
||||
Signature: option.SelfSignature,
|
||||
ClientId: option.ClientId,
|
||||
ClientEpoch: option.ClientEpoch,
|
||||
UntilNs: option.StopTsNs,
|
||||
ClientName: clientName,
|
||||
PathPrefix: pathPrefix,
|
||||
PathPrefixes: additionalPathPrefixes,
|
||||
Directories: directoriesToWatch,
|
||||
SinceNs: *lastTsNs,
|
||||
Signature: selfSignature,
|
||||
ClientId: clientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
UntilNs: untilTsNs,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribe: %v", err)
|
||||
@@ -84,7 +71,7 @@ func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn Proc
|
||||
}
|
||||
|
||||
if err := processEventFn(resp); err != nil {
|
||||
switch option.EventErrorType {
|
||||
switch eventErrorType {
|
||||
case TrivialOnError:
|
||||
glog.Errorf("process %v: %v", resp, err)
|
||||
case FatalOnError:
|
||||
@@ -100,7 +87,7 @@ func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn Proc
|
||||
glog.Errorf("process %v: %v", resp, err)
|
||||
}
|
||||
}
|
||||
option.StartTsNs = resp.TsNs
|
||||
*lastTsNs = resp.TsNs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,21 +34,9 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, p
|
||||
}
|
||||
|
||||
var clientEpoch int32
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: clientName,
|
||||
ClientId: s3a.randomClientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: prefix,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: directoriesToWatch,
|
||||
StartTsNs: lastTsNs,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.FatalOnError,
|
||||
}
|
||||
util.RetryForever("followIamChanges", func() error {
|
||||
clientEpoch++
|
||||
return pb.WithFilerClientFollowMetadata(s3a, metadataFollowOption, processEventFn)
|
||||
return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, directoriesToWatch, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
|
||||
}, func(err error) bool {
|
||||
glog.V(0).Infof("iam follow metadata changes: %v", err)
|
||||
return true
|
||||
@@ -77,7 +65,7 @@ func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, conte
|
||||
return nil
|
||||
}
|
||||
|
||||
// reload bucket metadata
|
||||
//reload bucket metadata
|
||||
func (s3a *S3ApiServer) onBucketMetadataChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
|
||||
if dir == s3a.option.BucketsPath {
|
||||
if newEntry != nil {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package s3_constants
|
||||
|
||||
// Amazon S3 predefined groups
|
||||
//Amazon S3 predefined groups
|
||||
var (
|
||||
GranteeGroupAllUsers = "http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user