Compare commits

..
2 Commits
Author SHA1 Message Date
Chris Lu a9e6db1a8e Merge branch 'master' into ftp 2021-01-21 00:47:27 -08:00
Chris Lu f0455dee68 WIP 2020-12-26 23:46:20 -08:00
837 changed files with 20896 additions and 63700 deletions
-4
View File
@@ -1,4 +0,0 @@
# These are supported funding model platforms
github: chrislusf
patreon: seaweedfs
@@ -9,7 +9,6 @@ assignees: ''
Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs
Please ask questions in https://github.com/chrislusf/seaweedfs/discussions
example of a good issue report:
https://github.com/chrislusf/seaweedfs/issues/1005
-10
View File
@@ -1,10 +0,0 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: gomod
directory: "/"
schedule:
interval: weekly
-10
View File
@@ -1,10 +0,0 @@
# What problem are we solving?
# How are we solving the problem?
# Checks
- [ ] I have added unit tests if possible.
- [ ] I will add related wiki document changes and link to this PR after merging.
-124
View File
@@ -1,124 +0,0 @@
name: "go: build dev binaries"
on:
push:
branches: [ master ]
permissions:
contents: read
jobs:
cleanup:
permissions:
contents: write # for mknejp/delete-release-assets to delete release assets
runs-on: ubuntu-latest
steps:
- name: Delete old release assets
uses: mknejp/delete-release-assets@a8aaab13272b1eaac16cc46dddd3f725b97ee05a # v1
with:
token: ${{ github.token }}
tag: dev
fail-if-no-assets: false
assets: |
weed-*
build_dev_linux_windows:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
needs: cleanup
runs-on: ubuntu-latest
strategy:
matrix:
goos: [linux, windows]
goarch: [amd64]
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
release_tag: dev
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-large-disk
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
release_tag: dev
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-normal-disk
asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
build_dev_darwin:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
needs: build_dev_linux_windows
runs-on: ubuntu-latest
strategy:
matrix:
goos: [darwin]
goarch: [amd64, arm64]
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
release_tag: dev
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-large-disk
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
release_tag: dev
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-normal-disk
asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
-59
View File
@@ -1,59 +0,0 @@
# This is a basic workflow to help you get started with Actions
name: "go: build versioned binaries for windows"
on:
push:
tags:
- '*'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_windows:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:
goos: [windows]
goarch: [amd64]
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
-59
View File
@@ -1,59 +0,0 @@
# This is a basic workflow to help you get started with Actions
name: "go: build versioned binaries for linux"
on:
push:
tags:
- '*'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_linux:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:
goos: [linux]
goarch: [amd64, arm, arm64]
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
-59
View File
@@ -1,59 +0,0 @@
# This is a basic workflow to help you get started with Actions
name: "go: build versioned binaries for darwin"
on:
push:
tags:
- '*'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_darwin:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:
goos: [darwin]
goarch: [amd64, arm64]
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
-59
View File
@@ -1,59 +0,0 @@
# This is a basic workflow to help you get started with Actions
name: "go: build versioned binaries for freebsd"
on:
push:
tags:
- '*'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_freebsd:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:
goos: [freebsd]
goarch: [amd64, arm, arm64]
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
-60
View File
@@ -1,60 +0,0 @@
# This is a basic workflow to help you get started with Actions
name: "go: build versioned binaries for linux with all tags"
on:
push:
tags:
- '*'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_linux:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:
goos: [linux]
goarch: [amd64]
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
build_flags: -tags elastic,ydb,gocdk,hdfs
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,hdfs
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full_large_disk"
+22
View File
@@ -0,0 +1,22 @@
name: Cleanup
on:
push:
branches: [ master ]
jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Delete old release assets
uses: mknejp/delete-release-assets@v1
with:
token: ${{ github.token }}
tag: dev
fail-if-no-assets: false
assets: |
weed-*
-43
View File
@@ -1,43 +0,0 @@
name: "Code Scanning - Action"
on:
pull_request:
jobs:
CodeQL-Build:
# CodeQL runs on ubuntu-latest, windows-latest, and macos-latest
runs-on: ubuntu-latest
permissions:
# required for all workflows
security-events: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
# Override language selection by uncommenting this and choosing your languages
with:
languages: go
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below).
- name: Autobuild
uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
# ✏️ If the Autobuild fails above, remove it and uncomment the following
# three lines and modify them (or add more) to build your code if your
# project uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
-66
View File
@@ -1,66 +0,0 @@
name: "docker: build dev containers"
on:
push:
branches: [ master ]
workflow_dispatch: []
permissions:
contents: read
jobs:
build-dev-containers:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: |
type=raw,value=dev
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
with:
buildkitd-flags: "--debug"
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
-
name: Build
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/amd64, linux/arm64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
-67
View File
@@ -1,67 +0,0 @@
name: "docker: build latest container"
on:
push:
tags:
- '*'
workflow_dispatch: []
permissions:
contents: read
jobs:
build-latest-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: |
type=raw,value=latest
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
with:
buildkitd-flags: "--debug"
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
-
name: Build
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
-57
View File
@@ -1,57 +0,0 @@
name: "docker: build release containers for normal volume"
on:
push:
tags:
- '*'
workflow_dispatch: []
permissions:
contents: read
jobs:
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
-59
View File
@@ -1,59 +0,0 @@
name: "docker: build release containers for large volume"
on:
push:
tags:
- '*'
workflow_dispatch: []
permissions:
contents: read
jobs:
build-large-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: TAGS=5BytesOffset
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
-58
View File
@@ -1,58 +0,0 @@
name: "docker: build release containers for rocksdb"
on:
push:
tags:
- '*'
workflow_dispatch: []
permissions:
contents: read
jobs:
build-large-release-container_rocksdb:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk_rocksdb
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.rocksdb_large
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
-58
View File
@@ -1,58 +0,0 @@
name: "docker: build release containers for all tags"
on:
push:
tags:
- '*'
workflow_dispatch: []
permissions:
contents: read
jobs:
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_full
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: TAGS=elastic,ydb,gocdk,hdfs
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
-58
View File
@@ -1,58 +0,0 @@
name: "docker: build release containers for all tags and large volume"
on:
push:
tags:
- '*'
workflow_dispatch: []
permissions:
contents: read
jobs:
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk_full
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,hdfs
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
-14
View File
@@ -1,14 +0,0 @@
name: 'Dependency Review'
on: [pull_request]
permissions:
contents: read
jobs:
dependency-review:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748
- name: 'Dependency Review'
uses: actions/dependency-review-action@a9c83d3af6b9031e20feba03b904645bb23d1dab
+9 -12
View File
@@ -1,4 +1,4 @@
name: "go: build binary"
name: Go
on:
push:
@@ -6,13 +6,6 @@ on:
pull_request:
branches: [ master ]
concurrency:
group: ${{ github.head_ref }}/go
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
@@ -21,20 +14,24 @@ jobs:
steps:
- name: Set up Go 1.x
uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v2
uses: actions/setup-go@v2
with:
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
uses: actions/checkout@v2
- name: Get dependencies
run: |
cd weed; go get -v -t -d ./...
if [ -f Gopkg.toml ]; then
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
dep ensure
fi
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite hdfs ydb" -v .
run: cd weed; go build -v .
- name: Test
run: cd weed; go test -tags "elastic gocdk sqlite hdfs ydb" -v ./...
run: cd weed; go test -v ./...
+62
View File
@@ -0,0 +1,62 @@
name: Release
on:
push:
branches: [ master ]
jobs:
build:
name: Build
runs-on: ubuntu-latest
strategy:
matrix:
goos: [linux, windows, darwin, freebsd ]
goarch: [amd64, arm]
exclude:
- goarch: arm
goos: darwin
- goarch: arm
goos: windows
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Wait for the deletion
uses: jakejarvis/wait-action@master
with:
time: '30s'
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV}
- name: Go Release Binaries
uses: wangyoucao577/go-release-action@v1.14
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
release_tag: dev
overwrite: true
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-large-disk
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries
uses: wangyoucao577/go-release-action@v1.14
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
release_tag: dev
overwrite: true
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-
asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
-4
View File
@@ -55,8 +55,6 @@ Temporary Items
# Mongo Explorer plugin:
# .idea/mongoSettings.xml
## vscode
.vscode
## File-based project format:
*.ipr
*.iws
@@ -77,8 +75,6 @@ com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
workspace/
test_data
build
target
+47
View File
@@ -0,0 +1,47 @@
sudo: false
language: go
go:
- 1.14.x
- 1.15.x
before_install:
- export PATH=/home/travis/gopath/bin:$PATH
install:
- export CGO_ENABLED="0"
- go env
script:
- env GO111MODULE=on go test ./weed/...
before_deploy:
- make release
deploy:
provider: releases
skip_cleanup: true
api_key:
secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
file:
- build/linux_arm.tar.gz
- build/linux_arm64.tar.gz
- build/linux_386.tar.gz
- build/linux_amd64.tar.gz
- build/linux_amd64_large_disk.tar.gz
- build/darwin_amd64.tar.gz
- build/darwin_amd64_large_disk.tar.gz
- build/windows_386.zip
- build/windows_amd64.zip
- build/windows_amd64_large_disk.zip
- build/freebsd_arm.tar.gz
- build/freebsd_amd64.tar.gz
- build/freebsd_386.tar.gz
- build/netbsd_arm.tar.gz
- build/netbsd_amd64.tar.gz
- build/netbsd_386.tar.gz
- build/openbsd_arm.tar.gz
- build/openbsd_amd64.tar.gz
- build/openbsd_386.tar.gz
on:
tags: true
repo: chrislusf/seaweedfs
go: 1.15.x
+140 -7
View File
@@ -1,11 +1,144 @@
BINARY = weed
BINARY = weed/weed
package = github.com/chrislusf/seaweedfs/weed
SOURCE_DIR = .
GO_FLAGS = #-v
SOURCE_DIR = ./weed/
all: install
appname := weed
install:
cd weed; go install
sources := $(wildcard *.go)
full_install:
cd weed; go install -tags "elastic gocdk sqlite hdfs ydb"
COMMIT ?= $(shell git rev-parse --short HEAD)
LDFLAGS ?= -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${COMMIT}
build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -static $(LDFLAGS)" -o build/$(appname)$(3) $(SOURCE_DIR)
tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static $(LDFLAGS)" -o build/$(appname)$(3) $(SOURCE_DIR)
tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
all: build
.PHONY : clean deps build linux release windows_build darwin_build linux_build bsd_build clean
clean:
go clean -i $(GO_FLAGS) $(SOURCE_DIR)
rm -f $(BINARY)
rm -rf build/
deps:
go get $(GO_FLAGS) -d $(SOURCE_DIR)
rm -rf /home/travis/gopath/src/github.com/coreos/etcd/vendor/golang.org/x/net/trace
rm -rf /home/travis/gopath/src/go.etcd.io/etcd/vendor/golang.org/x/net/trace
build: deps
go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY) $(SOURCE_DIR)
install: deps
go install $(GO_FLAGS) -ldflags "$(LDFLAGS)" $(SOURCE_DIR)
linux: deps
mkdir -p linux
GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o linux/$(BINARY) $(SOURCE_DIR)
release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_arm64_build 5_byte_darwin_build 5_byte_windows_build
##### LINUX BUILDS #####
5_byte_linux_build:
$(call build_large,linux,amd64,)
$(call tar_large,linux,amd64)
5_byte_darwin_build:
$(call build_large,darwin,amd64,)
$(call tar_large,darwin,amd64)
5_byte_windows_build:
$(call build_large,windows,amd64,.exe)
$(call zip_large,windows,amd64,.exe)
5_byte_arm_build: $(sources)
$(call build_large,linux,arm,)
$(call tar_large,linux,arm)
5_byte_arm64_build: $(sources)
$(call build_large,linux,arm64,)
$(call tar_large,linux,arm64)
linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
build/linux_386.tar.gz: $(sources)
$(call build,linux,386,)
$(call tar,linux,386)
build/linux_amd64.tar.gz: $(sources)
$(call build,linux,amd64,)
$(call tar,linux,amd64)
build/linux_arm.tar.gz: $(sources)
$(call build,linux,arm,)
$(call tar,linux,arm)
build/linux_arm64.tar.gz: $(sources)
$(call build,linux,arm64,)
$(call tar,linux,arm64)
##### DARWIN (MAC) BUILDS #####
darwin_build: build/darwin_amd64.tar.gz
build/darwin_amd64.tar.gz: $(sources)
$(call build,darwin,amd64,)
$(call tar,darwin,amd64)
##### WINDOWS BUILDS #####
windows_build: build/windows_386.zip build/windows_amd64.zip
build/windows_386.zip: $(sources)
$(call build,windows,386,.exe)
$(call zip,windows,386,.exe)
build/windows_amd64.zip: $(sources)
$(call build,windows,amd64,.exe)
$(call zip,windows,amd64,.exe)
##### BSD BUILDS #####
bsd_build: build/freebsd_arm.tar.gz build/freebsd_386.tar.gz build/freebsd_amd64.tar.gz \
build/netbsd_arm.tar.gz build/netbsd_386.tar.gz build/netbsd_amd64.tar.gz \
build/openbsd_arm.tar.gz build/openbsd_386.tar.gz build/openbsd_amd64.tar.gz
build/freebsd_386.tar.gz: $(sources)
$(call build,freebsd,386,)
$(call tar,freebsd,386)
build/freebsd_amd64.tar.gz: $(sources)
$(call build,freebsd,amd64,)
$(call tar,freebsd,amd64)
build/freebsd_arm.tar.gz: $(sources)
$(call build,freebsd,arm,)
$(call tar,freebsd,arm)
build/netbsd_386.tar.gz: $(sources)
$(call build,netbsd,386,)
$(call tar,netbsd,386)
build/netbsd_amd64.tar.gz: $(sources)
$(call build,netbsd,amd64,)
$(call tar,netbsd,amd64)
build/netbsd_arm.tar.gz: $(sources)
$(call build,netbsd,arm,)
$(call tar,netbsd,arm)
build/openbsd_386.tar.gz: $(sources)
$(call build,openbsd,386,)
$(call tar,openbsd,386)
build/openbsd_amd64.tar.gz: $(sources)
$(call build,openbsd,amd64,)
$(call tar,openbsd,amd64)
build/openbsd_arm.tar.gz: $(sources)
$(call build,openbsd,arm,)
$(call tar,openbsd,arm)
+31 -71
View File
@@ -1,14 +1,10 @@
# SeaweedFS
[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
[![Build Status](https://img.shields.io/github/workflow/status/chrislusf/seaweedfs/Go)](https://github.com/chrislusf/seaweedfs/actions/workflows/go.yml)
[![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
[![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
@@ -31,35 +27,30 @@ Your support will be really appreciated by me and other supporters!
</p>
-->
### Gold Sponsors
- [![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
![shuguang](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/shuguang.png)
---
- [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
- [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
- [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
- [SeaweedFS on Telegram](https://t.me/Seaweedfs)
- [SeaweedFS on Reddit](https://www.reddit.com/r/SeaweedFS/)
- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
- [SeaweedFS Introduction Slides 2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing)
- [SeaweedFS Introduction Slides 2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
Table of Contents
=================
* [Quick Start](#quick-start)
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
* [Quick Start with Single Binary](#quick-start-with-single-binary)
* [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws)
* [Introduction](#introduction)
* [Features](#features)
* [Additional Features](#additional-features)
* [Filer Features](#filer-features)
* [Example: Using Seaweed Object Store](#example-Using-Seaweed-Object-Store)
* [Architecture](#Object-Store-Architecture)
* [Architecture](#architecture)
* [Compared to Other File Systems](#compared-to-other-file-systems)
* [Compared to HDFS](#compared-to-hdfs)
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
@@ -72,18 +63,11 @@ Table of Contents
* [License](#license)
## Quick Start for S3 API on Docker ##
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
## Quick Start with Single Binary ##
## Quick Start ##
* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
## Quick Start SeaweedFS S3 on AWS ##
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it!
## Introduction ##
@@ -92,40 +76,24 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
1. to store billions of files!
2. to serve the files fast!
SeaweedFS started as an Object Store to handle small files efficiently.
Instead of managing all file metadata in a central master,
the central master only manages volumes on volume servers,
and these volume servers manage files and their metadata.
This relieves concurrency pressure from the central master and spreads file metadata into volume servers,
allowing faster file access (O(1), usually just one disk read operation).
SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages volumes on volume servers, and these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation).
There is only 40 bytes of disk storage overhead for each file's metadata.
It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity. What's more, the cloud storage access API cost is minimized. Faster and Cheaper than direct cloud storage!
SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
Also, SeaweedFS implements erasure coding with ideas from
[f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and has a lot of similarities with [Facebook’s Tectonic Filesystem](https://www.usenix.org/system/files/fast21-pan.pdf)
There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
On top of the object store, optional [Filer] can support directories and POSIX attributes.
Filer is a separate linearly-scalable stateless server with customizable metadata stores,
e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, YDB, etc.
SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf)
For any distributed key value stores, the large values can be offloaded to SeaweedFS.
With the fast access speed and linearly scalable capacity,
SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore].
On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, MemSql, TiDB, Etcd, CockroachDB, etc.
SeaweedFS can transparently integrate with the cloud.
With hot data on local cluster, and warm data on the cloud with O(1) access time,
SeaweedFS can achieve both fast local access time and elastic cloud storage capacity.
What's more, the cloud storage access API cost is minimized.
Faster and Cheaper than direct cloud storage!
For any distributed key value stores, the large values can be offloaded to SeaweedFS. With the fast access speed and linearly scalable capacity, SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore].
[Back to TOC](#table-of-contents)
## Additional Features ##
* Can choose no replication or different replication levels, rack and data center aware.
* Automatic master servers failover - no single point of failure (SPOF).
* Automatic Gzip compression depending on file MIME type.
* Automatic Gzip compression depending on file mime type.
* Automatic compaction to reclaim disk space after deletion or update.
* [Automatic entry TTL expiration][VolumeServerTTL].
* Any server with some disk spaces can add to the total storage space.
@@ -134,7 +102,6 @@ Faster and Cheaper than direct cloud storage!
* Support ETag, Accept-Range, Last-Modified, etc.
* Support in-memory/leveldb/readonly mode tuning for memory/performance balance.
* Support rebalancing the writable and readonly volumes.
* [Customizable Multiple Storage Tiers][TieredStorage]: Customizable storage disk types to balance performance and cost.
* [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data.
* [Erasure Coding for warm storage][ErasureCoding] Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
@@ -152,8 +119,6 @@ Faster and Cheaper than direct cloud storage!
* [WebDAV] accesses as a mapped drive on Mac and Windows, or from mobile devices.
* [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
* [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
## Kubernetes ##
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
@@ -167,7 +132,6 @@ Faster and Cheaper than direct cloud storage!
[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
[TieredStorage]: https://github.com/chrislusf/seaweedfs/wiki/Tiered-Storage
[CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier
[FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption
[FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores
@@ -176,9 +140,6 @@ Faster and Cheaper than direct cloud storage!
[ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
[FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
[KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
[CloudDrive]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Drive-Architecture
[GatewayToRemoteObjectStore]: https://github.com/chrislusf/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
[Back to TOC](#table-of-contents)
@@ -204,7 +165,7 @@ SeaweedFS uses HTTP REST operations to read, write, and delete. The responses ar
### Write File ###
To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server URL:
To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server url:
```
> curl http://localhost:9333/dir/assign
@@ -253,7 +214,7 @@ First look up the volume server's URLs by the file's volumeId:
Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depending on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.
Now you can take the public URL, render the URL or directly read from the volume server via URL:
Now you can take the public url, render the url or directly read from the volume server via url:
```
http://localhost:8080/3,01637037d6.jpg
@@ -332,7 +293,7 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
[Back to TOC](#table-of-contents)
## Object Store Architecture ##
## Architecture ##
Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
@@ -354,9 +315,9 @@ On each write request, the master server also generates a file key, which is a g
### Write and Read files ###
When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node URL) for the file. The client then contacts the volume node and POSTs the file content.
When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node url) for the file. The client then contacts the volume node and POSTs the file content.
When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node URL, volume node public URL), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node url, volume node public url), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
Please see the example for details on the write-read process.
@@ -410,7 +371,7 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa
* SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files.
* SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached.
* SeaweedFS Filer metadata store can be any well-known and proven data store, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd, YDB etc, and is easy to customize.
* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized.
* SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc.
| System | File Metadata | File Content Read| POSIX | REST API | Optimized for large number of small files |
@@ -446,13 +407,13 @@ Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more com
SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage.
Ceph, like SeaweedFS, is based on the object store RADOS. Ceph is rather complicated with mixed reviews.
Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
Ceph uses CRUSH hashing to automatically manage data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. Topology changes, such as adding new servers to increase capacity, will cause data migration with high IO cost to fit the CRUSH algorithm. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes is also as simple as it can be.
Ceph uses CRUSH hashing to automatically manage the data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes are also as simple as it can be.
SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read.
SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, YDB, to manage file directories. These stores are proven, scalable, and easier to manage.
SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage.
| SeaweedFS | comparable to Ceph | advantage |
| ------------- | ------------- | ---------------- |
@@ -497,7 +458,7 @@ Step 1: install go on your machine and setup the environment by following the in
https://golang.org/doc/install
make sure to define your $GOPATH
make sure you set up your $GOPATH
Step 2: checkout this repo:
@@ -507,7 +468,7 @@ git clone https://github.com/chrislusf/seaweedfs.git
Step 3: download, compile, and install the project by executing the following command
```bash
cd seaweedfs/weed && make install
make install
```
Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
@@ -534,7 +495,7 @@ Write 1 million 1KB file:
```
Concurrency Level: 16
Time taken for tests: 66.753 seconds
Completed requests: 1048576
Complete requests: 1048576
Failed requests: 0
Total transferred: 1106789009 bytes
Requests per second: 15708.23 [#/sec]
@@ -560,7 +521,7 @@ Randomly read 1 million files:
```
Concurrency Level: 16
Time taken for tests: 22.301 seconds
Completed requests: 1048576
Complete requests: 1048576
Failed requests: 0
Total transferred: 1106812873 bytes
Requests per second: 47019.38 [#/sec]
@@ -598,7 +559,6 @@ The text of this page is available for modification and reuse under the terms of
[Back to TOC](#table-of-contents)
## Stargazers over time
[![Stargazers over time](https://starchart.cc/chrislusf/seaweedfs.svg)](https://starchart.cc/chrislusf/seaweedfs)
## Stargazers over time ##
[![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)
+2 -8
View File
@@ -1,20 +1,14 @@
<h1 align="center">Sponsors &amp; Backers</h1>
- [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs).
<h2 align="center">Generous Backers ($50+)</h2>
- [4Sight Imaging](https://www.4sightimaging.com/)
- [Evercam Camera Management Software](https://evercam.io/)
- [Admiral](https://getadmiral.com)
<h2 align="center">Backers</h2>
- [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
- [Haravan - Ecommerce Platform](https://www.haravan.com)
- PeterCxy - Creator of Shelter App
- [Hive Games](https://playhive.com/)
- Flowm
- Yoni Nakache
- Catalin Constantin
- MingLi Yuan
- Leroy van Logchem
+49
View File
@@ -0,0 +1,49 @@
FROM alpine
RUN \
ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \
elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \
elif [ $(uname -m) == "aarch64" ]; then echo "arm64"; \
elif [ $(uname -m) == "armv7l" ]; then echo "arm"; \
elif [ $(uname -m) == "armv6l" ]; then echo "arm"; fi;) && \
echo "Building for $ARCH" 1>&2 && \
SUPERCRONIC_SHA1SUM=$(echo $ARCH | sed 's/386/e0126b0102b9f388ecd55714358e3ad60d0cebdb/g' | sed 's/amd64/5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85/g' | sed 's/arm64/e2714c43e7781bf1579c85aa61259245f56dbba1/g' | sed 's/arm/47481c3341bc3a1ae91a728e0cc63c8e6d3791ad/g') && \
SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-$ARCH && \
SUPERCRONIC=supercronic-linux-$ARCH && \
# Install SeaweedFS and Supercronic ( for cron job mode )
apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
apk add fuse && \
wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz") && \
tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \
curl -fsSLO "$SUPERCRONIC_URL" && \
echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \
chmod +x "$SUPERCRONIC" && \
mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
apk del build-dependencies && \
rm -rf /tmp/*
# volume server gprc port
EXPOSE 18080
# volume server http port
EXPOSE 8080
# filer server gprc port
EXPOSE 18888
# filer server http port
EXPOSE 8888
# master server shared gprc port
EXPOSE 19333
# master server shared http port
EXPOSE 9333
# s3 server http port
EXPOSE 8333
RUN mkdir -p /data/filerldb2
VOLUME /data
COPY filer.toml /etc/seaweedfs/filer.toml
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+4 -8
View File
@@ -1,17 +1,16 @@
FROM golang:1.18-alpine as builder
RUN apk add git g++ fuse
FROM frolvlad/alpine-glibc as builder
RUN apk add git go g++ fuse
RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
ARG BRANCH=${BRANCH:-master}
ARG TAGS
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
&& go install -ldflags "${LDFLAGS}"
FROM alpine AS final
LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/
COPY --from=builder /root/go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
@@ -31,13 +30,10 @@ EXPOSE 19333
EXPOSE 9333
# s3 server http port
EXPOSE 8333
# webdav server http port
EXPOSE 7333
RUN mkdir -p /data/filerldb2
VOLUME /data
WORKDIR /data
RUN chmod +x /entrypoint.sh
@@ -1,17 +1,16 @@
FROM gcc:11 as builder
FROM frolvlad/alpine-glibc as builder
RUN apk add git go g++ fuse
RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
ARG BRANCH=${BRANCH:-master}
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
&& apt-get update \
&& apt-get install -y golang-src \
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
&& go install -tags 5BytesOffset -ldflags "${LDFLAGS}"
FROM alpine AS final
LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/
COPY --from=builder /root/go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
@@ -31,14 +30,11 @@ EXPOSE 19333
EXPOSE 9333
# s3 server http port
EXPOSE 8333
# webdav server http port
EXPOSE 7333
RUN mkdir -p /data/filerldb2
VOLUME /data
WORKDIR /data
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
-3
View File
@@ -20,13 +20,10 @@ EXPOSE 19333
EXPOSE 9333
# s3 server http port
EXPOSE 8333
# webdav server http port
EXPOSE 7333
RUN mkdir -p /data/filerldb2
VOLUME /data
WORKDIR /data
RUN chmod +x /entrypoint.sh
-61
View File
@@ -1,61 +0,0 @@
FROM golang:1.18-buster as builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ENV ROCKSDB_VERSION v7.2.2
# build RocksDB
RUN cd /tmp && \
git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
cd rocksdb && \
PORTABLE=1 make static_lib && \
make install-static
ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
# build SeaweedFS
RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
ARG BRANCH=${BRANCH:-master}
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
FROM alpine AS final
LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
RUN apk add fuse snappy gflags
# volume server gprc port
EXPOSE 18080
# volume server http port
EXPOSE 8080
# filer server gprc port
EXPOSE 18888
# filer server http port
EXPOSE 8888
# master server shared gprc port
EXPOSE 19333
# master server shared http port
EXPOSE 9333
# s3 server http port
EXPOSE 8333
# webdav server http port
EXPOSE 7333
RUN mkdir -p /data/filer_rocksdb
VOLUME /data
WORKDIR /data
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
-31
View File
@@ -1,31 +0,0 @@
FROM ubuntu:20.04
RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
git \
sudo \
debianutils \
python3-pip \
python3-virtualenv \
python3-dev \
libevent-dev \
libffi-dev \
libxml2-dev \
libxslt-dev \
zlib1g-dev && \
DEBIAN_FRONTEND=noninteractive apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
git clone https://github.com/ceph/s3-tests.git /opt/s3-tests
WORKDIR /opt/s3-tests
RUN ./bootstrap
ENV \
NOSETESTS_EXCLUDE="" \
NOSETESTS_ATTR="" \
NOSETESTS_OPTIONS="" \
S3TEST_CONF="/s3test.conf"
ENTRYPOINT ["/bin/bash", "-c"]
CMD ["sleep 10 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
+6 -78
View File
@@ -4,94 +4,22 @@ all: gen
gen: dev
binary:
export SWCOMMIT=$(shell git rev-parse --short HEAD)
export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
cd ../weed; CGO_ENABLED=0 GOOS=linux go build -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
build: binary
build:
cd ../weed; GOOS=linux go build; mv weed ../docker/
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
rm ./weed
go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset
docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build .
go_build_large_disk:
docker build --build-arg TAGS=large_disk --no-cache -t chrislusf/seaweedfs:large_disk -f Dockerfile.go_build .
build_rocksdb:
docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
s3tests_build:
docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
dev: build
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_tls: build certstrap
ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_mount: build
docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
run_image: build
docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local
profile_mount: build
docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
docker-compose -f local-dev-compose.yml -p seaweedfs up
k8s: build
docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
docker-compose -f local-k8s-compose.yml -p seaweedfs up
dev_registry: build
docker-compose -f compose/local-registry-compose.yml -p seaweedfs up
dev_replicate: build
docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
dev_auditlog: build
docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
dev_nextcloud: build
docker-compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
docker-compose -f local-registry-compose.yml -p seaweedfs up
cluster: build
docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
2clusters: build
docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
2mount: build
docker-compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
hashicorp_raft: build
docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
s3tests: build s3tests_build
docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
filer_etcd: build
docker stack deploy -c compose/swarm-etcd.yml fs
test_etcd: build
docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
test_ydb: tags = ydb
test_ydb: build
export
docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up
docker-compose -f local-cluster-compose.yml -p seaweedfs up
clean:
rm ./weed
certstrap:
go get github.com/square/certstrap
certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true
View File
-4
View File
@@ -1,4 +0,0 @@
{
"fluent_port": 24224,
"fluent_host": "fluent"
}
-36
View File
@@ -1,36 +0,0 @@
version: '2'
services:
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333
- 9333:9333
- 19333:19333
- 8084:8080
- 18084:18080
- 8888:8888
- 18888:18888
- 8000:8000
command: "server -ip=s3 -filer -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.auditLogConfig=/etc/seaweedfs/fluent.json -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
volumes:
- ./fluent.json:/etc/seaweedfs/fluent.json
- ./s3.json:/etc/seaweedfs/s3.json
depends_on:
- fluent
fluent:
image: fluent/fluentd:v1.14
ports:
- 24224:24224
#s3tests:
# image: chrislusf/ceph-s3-tests:local
# volumes:
# - ./s3tests.conf:/opt/s3-tests/s3tests.conf
# environment:
# S3TEST_CONF: "s3tests.conf"
# NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
# NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
# NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
# depends_on:
# - s3
# - fluent
-88
View File
@@ -1,88 +0,0 @@
version: '2'
services:
master0:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
master1:
image: chrislusf/seaweedfs:local
ports:
- 9334:9334
- 19334:19334
command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
master2:
image: chrislusf/seaweedfs:local
ports:
- 9335:9335
- 19335:19335
command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
volume1:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
depends_on:
- master0
- master1
- master2
volume2:
image: chrislusf/seaweedfs:local
ports:
- 8082:8082
- 18082:18082
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
depends_on:
- master0
- master1
- master2
volume3:
image: chrislusf/seaweedfs:local
ports:
- 8083:8083
- 18083:18083
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
depends_on:
- master0
- master1
- master2
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8111:8111
command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
depends_on:
- master0
- master1
- master2
- volume1
- volume2
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333
command: '-v=9 s3 -filer="filer:8888"'
depends_on:
- master0
- master1
- master2
- volume1
- volume2
- filer
-28
View File
@@ -1,28 +0,0 @@
version: '2'
services:
server1:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
- 8084:8080
- 18084:18080
- 8888:8888
- 18888:18888
command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
volumes:
- ./master-cloud.toml:/etc/seaweedfs/master.toml
depends_on:
- server2
server2:
image: chrislusf/seaweedfs:local
ports:
- 9334:9333
- 19334:19333
- 8085:8080
- 18085:18080
- 8889:8888
- 18889:18888
- 8334:8333
command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
@@ -1,89 +0,0 @@
version: '2'
services:
master0:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
volumes:
- ./master/0:/data
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
master1:
image: chrislusf/seaweedfs:local
ports:
- 9334:9334
- 19334:19334
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
volumes:
- ./master/1:/data
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
master2:
image: chrislusf/seaweedfs:local
ports:
- 9335:9335
- 19335:19335
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
volumes:
- ./master/2:/data
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
volume1:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
depends_on:
- master0
- master1
volume2:
image: chrislusf/seaweedfs:local
ports:
- 8082:8082
- 18082:18082
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
depends_on:
- master0
- master1
volume3:
image: chrislusf/seaweedfs:local
ports:
- 8083:8083
- 18083:18083
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
depends_on:
- master0
- master1
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8111:8111
command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
depends_on:
- master0
- master1
- volume1
- volume2
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333
command: '-v=9 s3 -ip.bind="s3" -filer="filer:8888"'
depends_on:
- master0
- master1
- volume1
- volume2
- filer
@@ -1,50 +0,0 @@
version: '2'
services:
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
depends_on:
- master
s3:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8333:8333
command: '-v 1 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
depends_on:
- master
- volume
minio-gateway-s3:
image: minio/minio
ports:
- 9000:9000
command: 'minio gateway s3 http://s3:8333'
restart: on-failure
environment:
MINIO_ACCESS_KEY: "some_access_key1"
MINIO_SECRET_KEY: "some_secret_key1"
depends_on:
- s3
minio-warp:
image: minio/warp
command: 'mixed --duration=5m --obj.size=3mb --autoterm'
restart: on-failure
environment:
WARP_HOST: "minio-gateway-s3:9000"
WARP_ACCESS_KEY: "some_access_key1"
WARP_SECRET_KEY: "some_secret_key1"
depends_on:
- minio-gateway-s3
-46
View File
@@ -1,46 +0,0 @@
version: '2'
services:
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master"
volume:
image: chrislusf/seaweedfs:local
ports:
- 7455:8080
- 9325:9325
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
depends_on:
- master
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 9326:9326
command: 'filer -master="master:9333" -metricsPort=9326'
tty: true
stdin_open: true
depends_on:
- master
- volume
mount_1:
image: chrislusf/seaweedfs:local
privileged: true
entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1 -volumeServerAccess=filerProxy"'
depends_on:
- master
- volume
- filer
mount_2:
image: chrislusf/seaweedfs:local
privileged: true
entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1"'
depends_on:
- master
- volume
- filer
- mount_1
@@ -1,47 +0,0 @@
version: '2'
services:
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master"
volume:
image: chrislusf/seaweedfs:local
ports:
- 7455:8080
- 9325:9325
volumes:
- /Volumes/mobile_disk/99:/data
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
depends_on:
- master
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 9326:9326
volumes:
- /Volumes/mobile_disk/99:/data
command: 'filer -master="master:9333" -metricsPort=9326'
tty: true
stdin_open: true
depends_on:
- master
- volume
mount:
image: chrislusf/seaweedfs:local
privileged: true
cap_add:
- SYS_ADMIN
devices:
- fuse
volumes:
- /Volumes/mobile_disk/99:/data
entrypoint: '/bin/sh -c "mkdir -p t1 && weed -v=4 mount -filer=filer:8888 -dir=./t1 -cacheCapacityMB=0 -memprofile=/data/mount.mem.pprof"'
depends_on:
- master
- volume
- filer
@@ -1,44 +0,0 @@
version: '2'
services:
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master"
volume:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: "volume -mserver=master:9333 -port=8080 -ip=volume"
depends_on:
- master
s3:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8333:8333
command: '-v 9 filer -master="master:9333" -s3'
depends_on:
- master
- volume
nextcloud:
image: nextcloud:23.0.5-apache
environment:
- OBJECTSTORE_S3_HOST=s3
- OBJECTSTORE_S3_BUCKET=nextcloud
- OBJECTSTORE_S3_KEY=some_access_key1
- OBJECTSTORE_S3_SECRET=some_secret_key1
- OBJECTSTORE_S3_PORT=8333
- OBJECTSTORE_S3_SSL=false
- OBJECTSTORE_S3_USEPATH_STYLE=true
- SQLITE_DATABASE=nextcloud
- NEXTCLOUD_ADMIN_USER=admin
- NEXTCLOUD_ADMIN_PASSWORD=admin
ports:
- 80:80
depends_on:
- s3
@@ -1,61 +0,0 @@
version: '2'
services:
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master"
volume:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
depends_on:
- master
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
command: '-v=9 filer -master="master:9333"'
restart: on-failure
volumes:
- ./notification.toml:/etc/seaweedfs/notification.toml
depends_on:
- master
- volume
- rabbitmq
- replicate
environment:
RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/"
replicate:
image: chrislusf/seaweedfs:local
command: '-v=9 filer.replicate'
restart: on-failure
volumes:
- ./notification.toml:/etc/seaweedfs/notification.toml
- ./replication.toml:/etc/seaweedfs/replication.toml
depends_on:
- rabbitmq
environment:
RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/"
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333
command: 's3 -filer="filer:8888"'
depends_on:
- master
- volume
- filer
rabbitmq:
image: rabbitmq:3.8.10-management-alpine
ports:
- 5672:5672
- 15671:15671
- 15672:15672
environment:
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbit log_levels [{connection,error},{queue,debug}]"
-45
View File
@@ -1,45 +0,0 @@
version: '2'
services:
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=16"
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
volume:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
depends_on:
- master
s3:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8000:8000
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false'
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
depends_on:
- master
- volume
s3tests:
image: chrislusf/ceph-s3-tests:local
volumes:
- ./s3tests.conf:/opt/s3-tests/s3tests.conf
environment:
S3TEST_CONF: "s3tests.conf"
NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket|list_multipart_upload_owner|multipart_upload_small)"
depends_on:
- master
- volume
- s3
@@ -1,21 +0,0 @@
version: '3.9'
services:
node1:
image: chrislusf/seaweedfs:local
command: "server -master -volume -filer"
mount1:
image: chrislusf/seaweedfs:local
privileged: true
command: "mount -filer=node1:8888 -dir=/mnt -dirAutoCreate"
node2:
image: chrislusf/seaweedfs:local
ports:
- 7888:8888
command: "server -master -volume -filer"
mount2:
image: chrislusf/seaweedfs:local
privileged: true
command: "mount -filer=node2:8888 -dir=/mnt -dirAutoCreate"
sync:
image: chrislusf/seaweedfs:local
command: "-v=4 filer.sync -a=node1:8888 -b=node2:8888 -a.debug -b.debug"
-31
View File
@@ -1,31 +0,0 @@
# Put this file to one of the location, with descending priority
# ./master.toml
# $HOME/.seaweedfs/master.toml
# /etc/seaweedfs/master.toml
# this file is read by master
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
scripts = """
lock
ec.encode -fullPercent=95 -quietFor=1h
ec.rebuild -force
ec.balance -force
volume.balance -force
volume.fix.replication
unlock
"""
sleep_minutes = 17 # sleep minutes between each script execution
# configurations for tiered cloud storage
# old volumes are transparently moved to cloud for cost efficiency
[storage.backend]
[storage.backend.s3.default]
enabled = true
aws_access_key_id = "any" # if empty, loads from the shared credentials file (~/.aws/credentials).
aws_secret_access_key = "any" # if empty, loads from the shared credentials file (~/.aws/credentials).
region = "us-east-2"
bucket = "volume_bucket" # an existing bucket
endpoint = "http://server2:8333"
storage_class = "STANDARD_IA"
-17
View File
@@ -1,17 +0,0 @@
[notification.log]
# this is only for debugging perpose and does not work with "weed filer.replicate"
enabled = false
[notification.gocdk_pub_sub]
# The Go Cloud Development Kit (https://gocloud.dev).
# PubSub API (https://godoc.org/gocloud.dev/pubsub).
# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
enabled = true
# This URL will Dial the RabbitMQ server at the URL in the environment
# variable RABBIT_SERVER_URL and open the exchange "myexchange".
# The exchange must have already been created by some other means, like
# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then
# create binding myexchange => myqueue
topic_url = "rabbit://swexchange"
sub_url = "rabbit://swqueue"
-11
View File
@@ -1,11 +0,0 @@
[source.filer]
enabled = true
grpcAddress = "filer:18888"
# all files under this directory tree are replicated.
# this is not a directory on your hard drive, but on your filer.
# i.e., all files with this "prefix" are sent to notification message queue.
directory = "/buckets"
[sink.local_incremental]
enabled = true
directory = "/data"
-105
View File
@@ -1,105 +0,0 @@
{
"identities": [
{
"name": "anonymous",
"actions": [
"Read"
]
},
{
"name": "some_admin_user",
"credentials": [
{
"accessKey": "some_access_key1",
"secretKey": "some_secret_key1"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
},
{
"name": "s3_tests",
"credentials": [
{
"accessKey": "ABCDEFGHIJKLMNOPQRST",
"secretKey": "abcdefghijklmnopqrstuvwxyzabcdefghijklmn"
},
{
"accessKey": "0555b35654ad1656d804",
"secretKey": "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
},
{
"name": "s3_tests_alt",
"credentials": [
{
"accessKey": "NOPQRSTUVWXYZABCDEFG",
"secretKey": "nopqrstuvwxyzabcdefghijklmnabcdefghijklm"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
},
{
"name": "s3_tests_tenant",
"credentials": [
{
"accessKey": "HIJKLMNOPQRSTUVWXYZA",
"secretKey": "opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
},
{
"name": "some_read_only_user",
"credentials": [
{
"accessKey": "some_access_key2",
"secretKey": "some_secret_key2"
}
],
"actions": [
"Read"
]
},
{
"name": "some_normal_user",
"credentials": [
{
"accessKey": "some_access_key3",
"secretKey": "some_secret_key3"
}
],
"actions": [
"Read",
"List",
"Tagging",
"Write"
]
}
]
}
-70
View File
@@ -1,70 +0,0 @@
[DEFAULT]
## this section is just used for host, port and bucket_prefix
# host set for rgw in vstart.sh
host = s3
# port set for rgw in vstart.sh
port = 8000
## say "False" to disable TLS
is_secure = False
[fixtures]
## all the buckets created will start with this prefix;
## {random} will be filled with random characters to pad
## the prefix to 30 characters long, and avoid collisions
bucket prefix = yournamehere-{random}-
[s3 main]
# main display_name set in vstart.sh
display_name = M. Tester
# main user_idname set in vstart.sh
user_id = testid
# main email set in vstart.sh
email = tester@ceph.com
# zonegroup api_name for bucket location
api_name = default
## main AWS access key
access_key = 0555b35654ad1656d804
## main AWS secret key
secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
## replace with key id obtained when secret is created, or delete if KMS not tested
#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
[s3 alt]
# alt display_name set in vstart.sh
display_name = john.doe
## alt email set in vstart.sh
email = john.doe@example.com
# alt user_id set in vstart.sh
user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234
# alt AWS access key set in vstart.sh
access_key = NOPQRSTUVWXYZABCDEFG
# alt AWS secret key set in vstart.sh
secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm
[s3 tenant]
# tenant display_name set in vstart.sh
display_name = testx$tenanteduser
# tenant user_id set in vstart.sh
user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef
# tenant AWS secret key set in vstart.sh
access_key = HIJKLMNOPQRSTUVWXYZA
# tenant AWS secret key set in vstart.sh
secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab
# tenant email set in vstart.sh
email = tenanteduser@example.com
-84
View File
@@ -1,84 +0,0 @@
# 2021-01-30 16:25:30
version: '3.8'
services:
etcd:
image: gasparekatapy/etcd
networks:
- net
deploy:
mode: replicated
replicas: 3
master:
image: chrislusf/seaweedfs:local
environment:
WEED_MASTER_FILER_DEFAULT: "filer:8888"
WEED_MASTER_SEQUENCER_TYPE: "raft"
ports:
- "9333:9333"
- "19333:19333"
networks:
- net
command:
- 'master'
- '-resumeState=true'
- '-ip=master'
- '-port=9333'
deploy:
mode: replicated
replicas: 1
filer:
image: chrislusf/seaweedfs:local
environment:
WEED_LEVELDB2_ENABLED: "false"
WEED_ETCD_ENABLED: "true"
WEED_ETCD_SERVERS: "etcd:2379"
ports:
- target: 8888
published: 8888
protocol: tcp
mode: host
- target: 18888
published: 18888
protocol: tcp
mode: host
networks:
- net
command:
- 'filer'
- '-ip=filer'
- '-port=8888'
- '-port.readonly=28888'
- '-master=master:9333'
- '-disableDirListing=true'
deploy:
mode: replicated
replicas: 1
volume:
image: chrislusf/seaweedfs:local
ports:
- target: 8080
published: 8080
protocol: tcp
mode: host
- target: 18080
published: 18080
protocol: tcp
mode: host
networks:
- net
command:
- 'volume'
- '-mserver=master:9333'
- '-port=8080'
deploy:
mode: global
###########################################################################
networks:
net:
-61
View File
@@ -1,61 +0,0 @@
version: '2'
services:
etcd:
image: quay.io/coreos/etcd:v3.5.4
command: "etcd --advertise-client-urls http://etcd:2379 --listen-client-urls http://0.0.0.0:2379"
ports:
- 2379:2379
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
depends_on:
- master
s3:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8333:8333
command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
environment:
WEED_LEVELDB2_ENABLED: 'false'
WEED_ETCD_ENABLED: 'true'
WEED_ETCD_SERVERS: "http://etcd:2379"
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
depends_on:
- etcd
- master
- volume
registry:
image: registry:2
environment:
REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3
REGISTRY_LOG_LEVEL: "debug"
REGISTRY_STORAGE: "s3"
REGISTRY_STORAGE_S3_REGION: "us-east-1"
REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://s3:8333"
REGISTRY_STORAGE_S3_BUCKET: "registry"
REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1"
REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1"
REGISTRY_STORAGE_S3_V4AUTH: "true"
REGISTRY_STORAGE_S3_SECURE: "false"
REGISTRY_STORAGE_S3_SKIPVERIFY: "true"
REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/"
REGISTRY_STORAGE_DELETE_ENABLED: "true"
REGISTRY_STORAGE_REDIRECT_DISABLE: "true"
REGISTRY_VALIDATION_DISABLED: "true"
ports:
- 5001:5001
depends_on:
- s3
-35
View File
@@ -1,35 +0,0 @@
version: '2'
services:
ydb:
image: cr.yandex/yc/yandex-docker-local-ydb
ports:
- 2135:2135
- 8765:8765
- 2136:2136
environment:
- YDB_DEFAULT_LOG_LEVEL=DEBUG
- GRPC_TLS_PORT=2135
- GRPC_PORT=2136
- MON_PORT=8765
s3:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
- 8888:8888
- 8000:8000
- 18888:18888
command: "server -ip=s3 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false"
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
environment:
WEED_LEVELDB2_ENABLED: "false"
WEED_YDB_ENABLED: "true"
WEED_YDB_DSN: "grpc://ydb:2136/?database=local"
WEED_YDB_PREFIX: "seaweedfs"
YDB_ANONYMOUS_CREDENTIALS: 1
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
depends_on:
- ydb
-14
View File
@@ -1,14 +0,0 @@
WEED_GRPC_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
WEED_GRPC_ALLOWED_WILDCARD_DOMAIN=".dev"
WEED_GRPC_MASTER_CERT=/etc/seaweedfs/tls/master01.dev.crt
WEED_GRPC_MASTER_KEY=/etc/seaweedfs/tls/master01.dev.key
WEED_GRPC_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
WEED_GRPC_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
WEED_GRPC_FILER_CERT=/etc/seaweedfs/tls/filer01.dev.crt
WEED_GRPC_FILER_KEY=/etc/seaweedfs/tls/filer01.dev.key
WEED_GRPC_CLIENT_CERT=/etc/seaweedfs/tls/client01.dev.crt
WEED_GRPC_CLIENT_KEY=/etc/seaweedfs/tls/client01.dev.key
WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
+15 -11
View File
@@ -24,7 +24,7 @@ case "$1" in
'master')
ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
shift
exec /usr/bin/weed -logtostderr=true master $ARGS $@
exec /usr/bin/weed master $ARGS $@
;;
'volume')
@@ -33,7 +33,7 @@ case "$1" in
ARGS="-dir=/data"
fi
shift
exec /usr/bin/weed -logtostderr=true volume $ARGS $@
exec /usr/bin/weed volume $ARGS $@
;;
'server')
@@ -42,27 +42,31 @@ case "$1" in
ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
fi
shift
exec /usr/bin/weed -logtostderr=true server $ARGS $@
exec /usr/bin/weed server $ARGS $@
;;
'filer')
ARGS=""
shift
exec /usr/bin/weed -logtostderr=true filer $ARGS $@
exec /usr/bin/weed filer $ARGS $@
;;
's3')
ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
shift
exec /usr/bin/weed -logtostderr=true s3 $ARGS $@
exec /usr/bin/weed s3 $ARGS $@
;;
'shell')
ARGS="-cluster=$SHELL_CLUSTER -filer=$SHELL_FILER -filerGroup=$SHELL_FILER_GROUP -master=$SHELL_MASTER -options=$SHELL_OPTIONS"
shift
exec echo "$@" | /usr/bin/weed -logtostderr=true shell $ARGS
;;
'cronjob')
MASTER=${WEED_MASTER-localhost:9333}
FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab
BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
echo "$BALANCING_CRON_SCHEDULE" 'echo "volume.balance -c ALL -force" | weed shell -master='$MASTER >> /crontab
echo "Running Crontab:"
cat /crontab
exec supercronic /crontab
;;
*)
exec /usr/bin/weed $@
;;
-3
View File
@@ -1,3 +0,0 @@
[rocksdb]
enabled = true
dir = "/data/filer_rocksdb"
+53
View File
@@ -0,0 +1,53 @@
version: '2'
services:
master0:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m1"
master1:
image: chrislusf/seaweedfs:local
ports:
- 9334:9334
- 19334:19334
command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m2"
master2:
image: chrislusf/seaweedfs:local
ports:
- 9335:9335
- 19335:19335
command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m3"
volume:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume -publicUrl=localhost:8080'
depends_on:
- master0
- master1
- master2
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
command: 'filer -master="master0:9333,master1:9334,master2:9335"'
depends_on:
- master0
- master1
- master2
- volume
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333
command: 's3 -filer="filer:8888"'
depends_on:
- master0
- master1
- master2
- volume
- filer
@@ -6,50 +6,33 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "-v=1 master -ip=master"
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
command: "master -ip=master"
volume:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: "-v=1 volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
depends_on:
- master
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
filer:
image: chrislusf/seaweedfs:local
ports:
- 8111:8111
- 8888:8888
- 18888:18888
command: '-v=1 filer -master="master:9333" -iam'
command: 'filer -master="master:9333"'
depends_on:
- master
- volume
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333
command: '-v=1 s3 -filer="filer:8888"'
command: 's3 -filer="filer:8888"'
depends_on:
- master
- volume
- filer
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
mount:
image: chrislusf/seaweedfs:local
privileged: true
@@ -57,10 +40,6 @@ services:
- SYS_ADMIN
mem_limit: 4096m
command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=100 -concurrentWriters=128'
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
depends_on:
- master
- volume
@@ -15,18 +15,24 @@ services:
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
depends_on:
- master
s3:
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8333:8333
command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
command: 'filer -master="master:9333"'
depends_on:
- master
- volume
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333
command: '-v 9 s3 -filer="filer:8888"'
depends_on:
- master
- volume
- filer
minio:
image: minio/minio
ports:
+14 -13
View File
@@ -6,14 +6,14 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -ip.bind=0.0.0.0"
command: "master -ip=master"
volume:
image: chrislusf/seaweedfs # use a remote image
ports:
- 8080:8080
- 18080:18080
- 9325:9325
command: 'volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325'
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325'
depends_on:
- master
filer:
@@ -22,27 +22,28 @@ services:
- 8888:8888
- 18888:18888
- 9326:9326
command: 'filer -master="master:9333" -ip.bind=0.0.0.0 -metricsPort=9326'
command: 'filer -master="master:9333" -metricsPort=9326'
tty: true
stdin_open: true
depends_on:
- master
- volume
cronjob:
image: chrislusf/seaweedfs # use a remote image
command: 'cronjob'
environment:
# Run re-replication every 2 minutes
CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
WEED_MASTER: master:9333 # Default: localhost:9333
depends_on:
- master
- volume
s3:
image: chrislusf/seaweedfs # use a remote image
ports:
- 8333:8333
- 9327:9327
command: 's3 -filer="filer:8888" -ip.bind=0.0.0.0 -metricsPort=9327'
depends_on:
- master
- volume
- filer
webdav:
image: chrislusf/seaweedfs # use a remote image
ports:
- 7333:7333
command: 'webdav -filer="filer:8888"'
command: 's3 -filer="filer:8888" -metricsPort=9327'
depends_on:
- master
- volume
-9
View File
@@ -33,12 +33,3 @@ services:
- master
- volume
- filer
webdav:
image: chrislusf/seaweedfs:dev # use a remote dev image
ports:
- 7333:7333
command: 'webdav -filer="filer:8888"'
depends_on:
- master
- volume
- filer
+1 -1
View File
@@ -1,6 +1,6 @@
CREATE DATABASE IF NOT EXISTS seaweedfs;
CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret';
GRANT ALL PRIVILEGES ON seaweedfs.* TO 'seaweedfs'@'%';
GRANT ALL PRIVILEGES ON seaweedfs_fast.* TO 'seaweedfs'@'%';
FLUSH PRIVILEGES;
USE seaweedfs;
CREATE TABLE IF NOT EXISTS filemeta (
+73 -197
View File
@@ -1,225 +1,101 @@
module github.com/chrislusf/seaweedfs
go 1.18
go 1.12
require (
cloud.google.com/go v0.100.2 // indirect
cloud.google.com/go/pubsub v1.21.1
cloud.google.com/go/storage v1.22.1
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/OneOfOne/xxhash v1.2.8
github.com/Shopify/sarama v1.34.0
github.com/aws/aws-sdk-go v1.44.14
github.com/beorn7/perks v1.0.1 // indirect
cloud.google.com/go v0.44.3
github.com/Azure/azure-pipeline-go v0.2.2 // indirect
github.com/Azure/azure-storage-blob-go v0.8.0
github.com/OneOfOne/xxhash v1.2.2
github.com/Shopify/sarama v1.23.1
github.com/aws/aws-sdk-go v1.34.30
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash v1.1.0
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chrislusf/raft v1.0.8
github.com/colinmarc/hdfs/v2 v2.3.0
github.com/chrislusf/raft v1.0.4
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/disintegration/imaging v1.6.2
github.com/dustin/go-humanize v1.0.0
github.com/eapache/go-resiliency v1.2.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fclairamb/ftpserverlib v0.18.0
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/fclairamb/ftpserverlib v0.8.0
github.com/frankban/quicktest v1.7.2 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redsync/redsync/v4 v4.5.0
github.com/go-sql-driver/mysql v1.6.0
github.com/go-stack/stack v1.8.1 // indirect
github.com/go-zookeeper/zk v1.0.2 // indirect
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/uuid v1.3.0
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/go-redis/redis/v8 v8.4.4
github.com/go-sql-driver/mysql v1.5.0
github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
github.com/golang/protobuf v1.4.2
github.com/google/btree v1.0.0
github.com/google/uuid v1.1.1
github.com/gorilla/mux v1.7.4
github.com/gorilla/websocket v1.4.1 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.2
github.com/jinzhu/copier v0.3.5
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.15.1 // indirect
github.com/klauspost/reedsolomon v1.9.16
github.com/json-iterator/go v1.1.10
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/karlseguin/ccache/v2 v2.0.7
github.com/klauspost/compress v1.10.9 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect
github.com/klauspost/crc32 v1.2.0
github.com/klauspost/reedsolomon v1.9.2
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.6
github.com/linxGnu/grocksdb v1.7.1
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.3 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/olivere/elastic/v7 v7.0.32
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/peterh/liner v1.2.2
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/posener/complete v1.2.3
github.com/pquerna/cachecontrol v0.1.0
github.com/prometheus/client_golang v1.12.2
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/seaweedfs/goexif v2.0.0+incompatible
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/lib/pq v1.2.0
github.com/lunixbochs/vtclean v1.0.0 // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
github.com/mattn/go-runewidth v0.0.4 // indirect
github.com/olivere/elastic/v7 v7.0.19
github.com/peterh/liner v1.1.0
github.com/pierrec/lz4 v2.2.7+incompatible // indirect
github.com/prometheus/client_golang v1.3.0
github.com/rakyll/statik v0.1.7
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
github.com/seaweedfs/fuse v1.0.9
github.com/seaweedfs/goexif v1.0.2
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.12.0
github.com/streadway/amqp v1.0.0
github.com/stretchr/testify v1.7.1
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
github.com/tidwall/gjson v1.14.1
github.com/tidwall/match v1.1.1
github.com/tidwall/pretty v1.2.0 // indirect
github.com/spf13/viper v1.4.0
github.com/stretchr/testify v1.6.1
github.com/syndtr/goleveldb v1.0.0
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
github.com/tidwall/gjson v1.3.2
github.com/tidwall/match v1.0.1
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
github.com/valyala/bytebufferpool v1.0.0
github.com/viant/assertly v0.5.4 // indirect
github.com/viant/ptrie v0.3.0
github.com/viant/toolbox v0.33.2 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.1 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.etcd.io/etcd/client/v3 v3.5.4
go.mongodb.org/mongo-driver v1.9.1
go.opencensus.io v0.23.0 // indirect
gocloud.dev v0.25.0
gocloud.dev/pubsub/natspubsub v0.25.0
gocloud.dev/pubsub/rabbitpubsub v0.25.0
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
google.golang.org/api v0.81.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect
google.golang.org/grpc v1.46.2
google.golang.org/protobuf v1.28.0
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/cc/v3 v3.36.0 // indirect
modernc.org/ccgo/v3 v3.16.6 // indirect
modernc.org/libc v1.16.7 // indirect
modernc.org/mathutil v1.4.1 // indirect
modernc.org/memory v1.1.1 // indirect
modernc.org/opt v0.1.1 // indirect
modernc.org/sqlite v1.17.2
modernc.org/strutil v1.1.2
modernc.org/token v1.0.0 // indirect
github.com/willf/bitset v1.1.10 // indirect
github.com/willf/bloom v2.0.3+incompatible
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect
go.etcd.io/etcd v3.3.15+incompatible
go.mongodb.org/mongo-driver v1.3.2
gocloud.dev v0.16.0
gocloud.dev/pubsub/natspubsub v0.16.0
gocloud.dev/pubsub/rabbitpubsub v0.16.0
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114
google.golang.org/api v0.9.0
google.golang.org/appengine v1.6.2 // indirect
google.golang.org/grpc v1.29.1
google.golang.org/protobuf v1.24.0
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
)
require (
github.com/Jille/raft-grpc-transport v1.2.0
github.com/arangodb/go-driver v1.3.2
github.com/fluent/fluent-logger-golang v1.9.0
github.com/hanwen/go-fuse/v2 v2.1.0
github.com/hashicorp/raft v1.3.9
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3
)
// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
// replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft
require (
cloud.google.com/go/compute v1.6.1 // indirect
cloud.google.com/go/iam v0.3.0 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fclairamb/go-log v0.3.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/googleapis/go-type-adapters v1.0.0 // indirect
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.0.6 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
github.com/philhofer/fwd v1.1.1 // indirect
github.com/pierrec/lz4/v4 v4.1.14 // indirect
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.3.0 // indirect
github.com/tinylib/msgp v1.1.6 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc // indirect
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0 // indirect
lukechampine.com/uint128 v1.1.1 // indirect
)
// replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft
replace go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547
+595 -1164
View File
File diff suppressed because it is too large Load Diff
+23
View File
@@ -0,0 +1,23 @@
## SEAWEEDFS - helm chart (2.x)
### info:
* master/filer/volume are stateful sets with anti-affinity on the hostname,
so your deployment will be spread/HA.
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
and backup/HA memsql can provide.
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
with ENV.
* cert config exists and can be enabled, but not been tested.
### current instances config (AIO):
1 instance for each type (master/filer/volume/s3)
instances need node labels:
* sw-volume: true (for volume instance, specific tag)
* sw-backend: true (for all others, as they less resource demanding)
you can update the replicas count for each node type in values.yaml,
need to add more nodes with the corresponding label.
most of the configuration are available through values.yaml
-48
View File
@@ -1,48 +0,0 @@
## SEAWEEDFS - helm chart (2.x)
### info:
* master/filer/volume are stateful sets with anti-affinity on the hostname,
so your deployment will be spread/HA.
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
and backup/HA memsql can provide.
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
with ENV.
* cert config exists and can be enabled, but not been tested.
### prerequisites
kubernetes node have labels which help to define which node(Host) will run which pod.
s3/filer/master needs the label **sw-backend=true**
volume need the label **sw-volume=true**
to label a node to be able to run all pod types in k8s:
```
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
```
on production k8s deployment you will want each pod to have a different host,
especially the volume server & the masters, currently all pods (master/volume/filer)
have anti-affinity rule to disallow running multiple pod type on the same host.
if you still want to run multiple pods of the same type (master/volume/filer) on the same host
please set/update the corresponding affinity rule in values.yaml to an empty one:
```affinity: ""```
### PVC - storage class ###
on the volume stateful set added support for K8S PVC, currently example
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
https://github.com/rancher/local-path-provisioner
you can use ANY storage class you like, just update the correct storage-class
for your deployment.
### current instances config (AIO):
1 instance for each type (master/filer+s3/volume)
you can update the replicas count for each node type in values.yaml,
need to add more nodes with the corresponding labels.
most of the configuration are available through values.yaml
-67
View File
@@ -1,67 +0,0 @@
{{- if .Values.filer.ingress.enabled }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1beta1
{{- else }}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: ingress-{{ template "seaweedfs.name" . }}-filer
namespace: {{ .Release.Namespace }}
annotations:
{{ omit .Values.filer.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }}
spec:
ingressClassName: {{ .Values.filer.ingress.className | quote }}
rules:
- http:
paths:
- path: /sw-filer/?(.*)
pathType: ImplementationSpecific
backend:
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
service:
name: {{ template "seaweedfs.name" . }}-filer
port:
number: {{ .Values.filer.port }}
#name:
{{- else }}
serviceName: {{ template "seaweedfs.name" . }}-filer
servicePort: {{ .Values.filer.port }}
{{- end }}
{{- end }}
---
{{- if .Values.master.ingress.enabled }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1beta1
{{- else }}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: ingress-{{ template "seaweedfs.name" . }}-master
namespace: {{ .Release.Namespace }}
annotations:
{{ omit .Values.master.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }}
spec:
ingressClassName: {{ .Values.master.ingress.className | quote }}
rules:
- http:
paths:
- path: /sw-master/?(.*)
pathType: ImplementationSpecific
backend:
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
service:
name: {{ template "seaweedfs.name" . }}-master
port:
number: {{ .Values.master.port }}
#name:
{{- else }}
serviceName: {{ template "seaweedfs.name" . }}-master
servicePort: {{ .Values.master.port }}
{{- end }}
{{- end }}
@@ -1,29 +0,0 @@
{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation) }}
{{- $access_key_admin := randAlphaNum 16 -}}
{{- $secret_key_admin := randAlphaNum 32 -}}
{{- $access_key_read := randAlphaNum 16 -}}
{{- $secret_key_read := randAlphaNum 32 -}}
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: seaweedfs-s3-secret
namespace: {{ .Release.Namespace }}
annotations:
"helm.sh/resource-policy": keep
"helm.sh/hook": "pre-install"
stringData:
admin_access_key_id: {{ $access_key_admin }}
admin_secret_access_key: {{ $secret_key_admin }}
read_access_key_id: {{ $access_key_read }}
read_secret_access_key: {{ $secret_key_read }}
seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"{{ $access_key_read }}","secretKey":"{{ $secret_key_read }}"}],"actions":["Read"]}]}'
{{- if .Values.filer.s3.auditLogConfig }}
filer_s3_auditLogConfig.json: |
{{ toJson .Values.filer.s3.auditLogConfig | nindent 4 }}
{{- end }}
{{- if .Values.s3.auditLogConfig }}
s3_auditLogConfig.json: |
{{ toJson .Values.s3.auditLogConfig | nindent 4 }}
{{- end }}
{{- end }}
@@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "3.09"
version: "3.09"
appVersion: "2.21"
version: 2.21
@@ -113,24 +113,16 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{- end -}}
{{/* check if any PVC exists */}}
{{- define "volume.pvc_exists" -}}
{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
{{- printf "true" -}}
{{/* Return the proper cronjob image */}}
{{- define "cronjob.image" -}}
{{- if .Values.cronjob.imageOverride -}}
{{- $imageOverride := .Values.cronjob.imageOverride -}}
{{- printf "%s" $imageOverride -}}
{{- else -}}
{{- printf "false" -}}
{{- end -}}
{{- end -}}
{{/* check if any HostPath exists */}}
{{- define "volume.hostpath_exists" -}}
{{- if or (or (eq .Values.volume.data.type "hostPath") (and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "hostPath") -}}
{{- printf "true" -}}
{{- else -}}
{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
{{- printf "true" -}}
{{- else -}}
{{- printf "false" -}}
{{- end -}}
{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
{{- $repositoryName := .Values.image.repository | toString -}}
{{- $name := .Values.global.imageName | toString -}}
{{- $tag := .Chart.AppVersion | toString -}}
{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
{{- end -}}
{{- end -}}
+55
View File
@@ -0,0 +1,55 @@
{{- if .Values.cronjob }}
{{- if .Values.cronjob.enabled }}
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: {{ include "seaweedfs.fullname" . }}-cronjob
spec:
schedule: "{{ .Values.cronjob.schedule }}"
startingDeadlineSeconds: 200
concurrencyPolicy: Forbid
failedJobsHistoryLimit: 2
successfulJobsHistoryLimit: 2
jobTemplate:
spec:
backoffLimit: 2
template:
spec:
{{- with .Values.cronjob.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.cronjob.tolerations }}
tolerations:
{{- toYaml . | nindent 12 }}
{{- end }}
restartPolicy: OnFailure
containers:
- name: shell
image: {{ template "cronjob.image" . }}
imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
resources:
{{- toYaml .Values.cronjob.resources| nindent 16 }}
command:
- sh
- -c
- |
set -ex
echo -e "lock\n\
volume.balance -force\
{{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\
{{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\
volume.fix.replication\nunlock\n" | \
/usr/bin/weed shell \
{{- if .Values.cronjob.master }}
-master {{ .Values.cronjob.master }} \
{{- else }}
-master {{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc:{{ .Values.master.port }} \
{{- end }}
{{- if .Values.cronjob.filer }}
-filer {{ .Values.cronjob.filer }}
{{- else }}
-filer {{ template "seaweedfs.name" . }}-filer.{{ .Release.Namespace }}.svc:{{ .Values.filer.port }}
{{- end }}
{{- end }}
{{- end }}
@@ -28,4 +28,4 @@ spec:
{{- end }}
selector:
app: {{ template "seaweedfs.name" . }}
component: filer
component: filer
@@ -1,5 +1,4 @@
{{- if .Values.filer.metricsPort }}
{{- if .Values.global.monitoring.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
@@ -16,5 +15,4 @@ spec:
selector:
app: {{ template "seaweedfs.name" . }}
component: filer
{{- end }}
{{- end }}
@@ -133,37 +133,14 @@ spec:
-encryptVolumeData \
{{- end }}
-ip=${POD_IP} \
{{- if .Values.filer.s3.enabled }}
-s3 \
-s3.port={{ .Values.filer.s3.port }} \
{{- if .Values.filer.s3.domainName }}
-s3.domainName={{ .Values.filer.s3.domainName }} \
{{- end }}
{{- if .Values.global.enableSecurity }}
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
{{- end }}
{{- if .Values.filer.s3.allowEmptyFolder }}
-s3.allowEmptyFolder={{ .Values.filer.s3.allowEmptyFolder }} \
{{- end }}
{{- if .Values.filer.s3.enableAuth }}
-s3.config=/etc/sw/seaweedfs_s3_config \
{{- end }}
{{- if .Values.filer.s3.auditLogConfig }}
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
{{- end }}
{{- if gt (.Values.filer.replicas | int) 1 }}
-peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' ) \
{{- end }}
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
{{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}
volumeMounts:
- name: seaweedfs-filer-log-volume
mountPath: "/logs/"
- mountPath: /etc/sw
name: config-users
readOnly: true
{{- if .Values.filer.enablePVC }}
- name: data-filer
mountPath: /data
{{- end }}
{{- if .Values.global.enableSecurity }}
- name: security-config
readOnly: true
@@ -186,6 +163,7 @@ spec:
mountPath: /usr/local/share/ca-certificates/client/
{{- end }}
{{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
{{- end }}
ports:
- containerPort: {{ .Values.filer.port }}
name: swfs-filer
@@ -220,13 +198,6 @@ spec:
hostPath:
path: /storage/logs/seaweedfs/filer
type: DirectoryOrCreate
- name: db-schema-config-volume
configMap:
name: seaweedfs-db-init-config
- name: config-users
secret:
defaultMode: 420
secretName: seaweedfs-s3-secret
{{- if .Values.global.enableSecurity }}
- name: security-config
configMap:
@@ -252,18 +223,16 @@ spec:
nodeSelector:
{{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
{{- end }}
{{- if .Values.filer.enablePVC }}
volumeClaimTemplates:
- metadata:
name: data-filer
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.filer.storage }}
{{- if .Values.filer.storageClass }}
storageClassName: {{ .Values.filer.storageClass }}
{{- end }}
{{- end }}
{{/* volumeClaimTemplates:*/}}
{{/* - metadata:*/}}
{{/* name: data-{{ .Release.Namespace }}*/}}
{{/* spec:*/}}
{{/* accessModes:*/}}
{{/* - ReadWriteOnce*/}}
{{/* resources:*/}}
{{/* requests:*/}}
{{/* storage: {{ .Values.filer.storage }}*/}}
{{/* {{- if .Values.filer.storageClass }}*/}}
{{/* storageClassName: {{ .Values.filer.storageClass }}*/}}
{{/* {{- end }}*/}}
{{- end }}
+59
View File
@@ -0,0 +1,59 @@
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: ingress-{{ template "seaweedfs.name" . }}-filer
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/auth-type: "basic"
nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/rewrite-target: /$1
nginx.ingress.kubernetes.io/use-regex: "true"
nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
sub_filter '="/' '="./'; #make absolute paths to relative
sub_filter '=/' '=./';
sub_filter '/seaweedfsstatic' './seaweedfsstatic';
sub_filter_once off;
spec:
rules:
- http:
paths:
- path: /sw-filer/?(.*)
backend:
serviceName: {{ template "seaweedfs.name" . }}-filer
servicePort: {{ .Values.filer.port }}
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: ingress-{{ template "seaweedfs.name" . }}-master
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/auth-type: "basic"
nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/rewrite-target: /$1
nginx.ingress.kubernetes.io/use-regex: "true"
nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
sub_filter '="/' '="./'; #make absolute paths to relative
sub_filter '=/' '=./';
sub_filter '/seaweedfsstatic' './seaweedfsstatic';
sub_filter_once off;
spec:
rules:
- http:
paths:
- path: /sw-master/?(.*)
backend:
serviceName: {{ template "seaweedfs.name" . }}-master
servicePort: {{ .Values.master.port }}
@@ -10,6 +10,7 @@ metadata:
heritage: {{ .Release.Service }}
release: {{ .Release.Name }}
spec:
serviceName: {{ template "seaweedfs.name" . }}-s3
replicas: {{ .Values.s3.replicas }}
selector:
matchLabels:
@@ -89,19 +90,10 @@ spec:
{{- if .Values.s3.allowEmptyFolder }}
-allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \
{{- end }}
{{- if .Values.s3.enableAuth }}
-config=/etc/sw/seaweedfs_s3_config \
{{- end }}
{{- if .Values.s3.auditLogConfig }}
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
{{- end }}
-filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
volumeMounts:
- name: logs
mountPath: "/logs/"
- mountPath: /etc/sw
name: config-users
readOnly: true
{{- if .Values.global.enableSecurity }}
- name: security-config
readOnly: true
@@ -129,7 +121,7 @@ spec:
name: swfs-s3
readinessProbe:
httpGet:
path: /status
path: /
port: {{ .Values.s3.port }}
scheme: HTTP
initialDelaySeconds: 15
@@ -139,7 +131,7 @@ spec:
timeoutSeconds: 10
livenessProbe:
httpGet:
path: /status
path: /
port: {{ .Values.s3.port }}
scheme: HTTP
initialDelaySeconds: 20
@@ -152,10 +144,6 @@ spec:
{{ tpl .Values.s3.resources . | nindent 12 | trim }}
{{- end }}
volumes:
- name: config-users
secret:
defaultMode: 420
secretName: seaweedfs-s3-secret
{{- if eq .Values.s3.logs.type "hostPath" }}
- name: logs
hostPath:
@@ -9,15 +9,15 @@ metadata:
spec:
ports:
- name: "swfs-s3"
port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
targetPort: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
port: {{ .Values.s3.port }}
targetPort: {{ .Values.s3.port }}
protocol: TCP
{{- if and .Values.s3.enabled .Values.s3.metricsPort }}
- name: "metrics"
{{- if .Values.s3.metricsPort }}
- name: "swfs-s3-metrics"
port: {{ .Values.s3.metricsPort }}
targetPort: {{ .Values.s3.metricsPort }}
protocol: TCP
{{- end }}
selector:
app: {{ template "seaweedfs.name" . }}
component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }}
component: s3
@@ -1,5 +1,4 @@
{{- if .Values.s3.metricsPort }}
{{- if .Values.global.monitoring.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
@@ -16,5 +15,4 @@ spec:
selector:
app: {{ template "seaweedfs.name" . }}
component: s3
{{- end }}
{{- end }}
@@ -1,7 +1,7 @@
#hack for delete pod master after migration
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: seaweefds-rw-cr
rules:
@@ -16,7 +16,7 @@ metadata:
namespace: {{ .Release.Namespace }}
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: system:serviceaccount:seaweefds-rw-sa:default
subjects:
@@ -26,4 +26,4 @@ subjects:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: seaweefds-rw-cr
name: seaweefds-rw-cr
@@ -23,6 +23,6 @@ spec:
targetPort: {{ .Values.volume.metricsPort }}
protocol: TCP
{{- end }}
selector:
selector:
app: {{ template "seaweedfs.name" . }}
component: volume
@@ -1,5 +1,4 @@
{{- if .Values.volume.metricsPort }}
{{- if .Values.global.monitoring.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
@@ -16,5 +15,4 @@ spec:
selector:
app: {{ template "seaweedfs.name" . }}
component: volume
{{- end }}
{{- end }}
@@ -40,24 +40,11 @@ spec:
imagePullSecrets:
- name: {{ .Values.global.imagePullSecrets }}
{{- end }}
terminationGracePeriodSeconds: 150
terminationGracePeriodSeconds: 10
{{- if .Values.volume.priorityClassName }}
priorityClassName: {{ .Values.volume.priorityClassName | quote }}
{{- end }}
enableServiceLinks: false
{{- if .Values.volume.dir_idx }}
initContainers:
- name: seaweedfs-vol-move-idx
image: {{ template "volume.image" . }}
imagePullPolicy: {{ .Values.global.pullPolicy | default "IfNotPresent" }}
command: [ '/bin/sh', '-c' ]
args: ['if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;']
volumeMounts:
- name: idx
mountPath: {{ .Values.volume.dir_idx }}
- name: data
mountPath: {{ .Values.volume.dir }}
{{- end }}
containers:
- name: seaweedfs
image: {{ template "volume.image" . }}
@@ -110,7 +97,7 @@ spec:
-dataCenter={{ .Values.volume.dataCenter }} \
{{- end }}
-ip.bind={{ .Values.volume.ipBind }} \
-readMode={{ .Values.volume.readMode }} \
-read.redirect={{ .Values.volume.readRedirect }} \
{{- if .Values.volume.whiteList }}
-whiteList={{ .Values.volume.whiteList }} \
{{- end }}
@@ -131,13 +118,9 @@ spec:
-compactionMBps={{ .Values.volume.compactionMBps }} \
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
volumeMounts:
- name: data
mountPath: "{{ .Values.volume.dir }}/"
{{- if .Values.volume.dir_idx }}
- name: idx
mountPath: "{{ .Values.volume.dir_idx }}/"
{{- end }}
- name: logs
- name: seaweedfs-volume-storage
mountPath: "/data/"
- name: seaweedfs-volume-log-volume
mountPath: "/logs/"
{{- if .Values.global.enableSecurity }}
- name: security-config
@@ -190,27 +173,15 @@ spec:
resources:
{{ tpl .Values.volume.resources . | nindent 12 | trim }}
{{- end }}
{{- $hostpath_exists := include "volume.hostpath_exists" . -}}
{{- if $hostpath_exists }}
volumes:
{{- if eq .Values.volume.data.type "hostPath" }}
- name: data
hostPath:
path: /storage/object_store/
type: DirectoryOrCreate
{{- end }}
{{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
- name: idx
hostPath:
path: /ssd/seaweedfs-volume-idx/
type: DirectoryOrCreate
{{- end }}
{{- if eq .Values.volume.logs.type "hostPath" }}
- name: logs
- name: seaweedfs-volume-log-volume
hostPath:
path: /storage/logs/seaweedfs/volume
type: DirectoryOrCreate
{{- end }}
- name: seaweedfs-volume-storage
hostPath:
path: /storage/object_store/
type: DirectoryOrCreate
{{- if .Values.global.enableSecurity }}
- name: security-config
configMap:
@@ -234,43 +205,8 @@ spec:
{{- if .Values.volume.extraVolumes }}
{{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
{{- end }}
{{- end }}
{{- if .Values.volume.nodeSelector }}
nodeSelector:
{{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
{{- end }}
{{- $pvc_exists := include "volume.pvc_exists" . -}}
{{- if $pvc_exists }}
volumeClaimTemplates:
{{- if eq .Values.volume.data.type "persistentVolumeClaim"}}
- metadata:
name: data
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: {{ .Values.volume.data.storageClass }}
resources:
requests:
storage: {{ .Values.volume.data.size }}
{{- end }}
{{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }}
- metadata:
name: idx
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: {{ .Values.volume.idx.storageClass }}
resources:
requests:
storage: {{ .Values.volume.idx.size }}
{{- end }}
{{- if eq .Values.volume.logs.type "persistentVolumeClaim" }}
- metadata:
name: logs
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: {{ .Values.volume.logs.storageClass }}
resources:
requests:
storage: {{ .Values.volume.logs.size }}
{{- end }}
{{- end }}
{{- end }}
@@ -4,6 +4,7 @@ global:
registry: ""
repository: ""
imageName: chrislusf/seaweedfs
# imageTag: "2.21" - started using {.Chart.appVersion}
imagePullPolicy: IfNotPresent
imagePullSecrets: imagepullsecret
restartPolicy: Always
@@ -41,7 +42,8 @@ master:
grpcPort: 19333
ipBind: "0.0.0.0"
volumePreallocate: false
volumeSizeLimitMB: 1000
#Master stops directing writes to oversized volumes
volumeSizeLimitMB: 30000
loggingOverrideLevel: null
#number of seconds between heartbeats, default 5
pulseSeconds: null
@@ -108,26 +110,6 @@ master:
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
priorityClassName: ""
ingress:
enabled: false
className: "nginx"
annotations:
nginx.ingress.kubernetes.io/auth-type: "basic"
nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/rewrite-target: /$1
nginx.ingress.kubernetes.io/use-regex: "true"
nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
sub_filter '="/' '="./'; #make absolute paths to relative
sub_filter '=/' '=./';
sub_filter '/seaweedfsstatic' './seaweedfsstatic';
sub_filter_once off;
extraEnvironmentVars:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
@@ -154,33 +136,15 @@ volume:
# limit file size to avoid out of memory, default 256mb
fileSizeLimitMB: null
# minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
minFreeSpacePercent: 7
minFreeSpacePercent: 1
# can use ANY storage-class , example with local-path-provisner
# data:
# type: "persistentVolumeClaim"
# size: "24Ti"
# storageClass: "local-path-provisioner"
data:
type: "hostPath"
size: ""
storageClass: ""
idx:
type: "hostPath"
size: ""
storageClass: ""
logs:
type: "hostPath"
size: ""
storageClass: ""
# limit background compaction or copying speed in mega bytes per second
compactionMBps: "50"
# Directories to store data files. dir[,dir]... (default "/tmp")
dir: "/data"
# Directories to store index files. dir[,dir]... (default is the same as "dir")
# Directories to store index files. dir[,dir]... (default "/tmp")
dir_idx: null
# Maximum numbers of volumes, count[,count]...
@@ -193,8 +157,8 @@ volume:
# Volume server's data center name
dataCenter: null
# Redirect moved or non-local volumes. (default proxy)
readMode: proxy
# Redirect moved or non-local volumes. (default true)
readRedirect: true
# Comma separated Ip addresses having write permission. No limit if empty.
whiteList: null
@@ -272,12 +236,12 @@ filer:
# Limit sub dir listing size (default 100000)
dirListLimit: 100000
# Turn off directory listing
disableDirListing: false
# Disable http request, only gRpc operations are allowed
disableHttp: false
# enablePVC will create a pvc for filer for data persistence.
enablePVC: false
# storage and storageClass are the settings for configuring stateful
# storage for the master pods. storage should be set to the disk size of
# the attached volume. storageClass is the class of storage which defaults
@@ -328,26 +292,6 @@ filer:
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
priorityClassName: ""
ingress:
enabled: false
className: "nginx"
annotations:
nginx.ingress.kubernetes.io/auth-type: "basic"
nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/rewrite-target: /$1
nginx.ingress.kubernetes.io/use-regex: "true"
nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
sub_filter '="/' '="./'; #make absolute paths to relative
sub_filter '=/' '=./';
sub_filter '/seaweedfsstatic' './seaweedfsstatic';
sub_filter_once off;
# extraEnvVars is a list of extra enviroment variables to set with the stateful set.
extraEnvironmentVars:
WEED_MYSQL_ENABLED: "true"
@@ -360,7 +304,6 @@ filer:
WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS: "600"
# enable usage of memsql as filer backend
WEED_MYSQL_INTERPOLATEPARAMS: "true"
# if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
WEED_LEVELDB2_ENABLED: "false"
# with http DELETE, by default the filer would check whether a folder is empty.
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
@@ -368,20 +311,8 @@ filer:
# directories under this folder will be automatically creating a separate bucket
WEED_FILER_BUCKETS_FOLDER: "/buckets"
s3:
enabled: true
port: 8333
#allow empty folders
allowEmptyFolder: false
# Suffix of the host name, {bucket}.{domainName}
domainName: ""
# enable user & permission to s3 (need to inject to all services)
enableAuth: false
skipAuthSecretCreation: false
auditLogConfig: {}
s3:
enabled: false
enabled: true
repository: null
imageName: null
imageTag: null
@@ -392,10 +323,6 @@ s3:
loggingOverrideLevel: null
#allow empty folders
allowEmptyFolder: true
# enable user & permission to s3 (need to inject to all services)
enableAuth: false
skipAuthSecretCreation: false
auditLogConfig: {}
# Suffix of the host name, {bucket}.{domainName}
domainName: ""
@@ -431,6 +358,18 @@ s3:
size: ""
storageClass: ""
cronjob:
enabled: false
schedule: "*/7 * * * *"
resources: null
# balance all volumes among volume servers
# ALL|EACH_COLLECTION|<collection_name>
collection: ""
master: ""
filer: ""
tolerations: ""
nodeSelector: |
sw-backend: "true"
certificates:
commonName: "SeaweedFS CA"

Some files were not shown because too many files have changed in this diff Show More