mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-19 04:55:53 +00:00
Compare commits
132
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87021a1460 | ||
|
|
0748214c8e | ||
|
|
e19940abca | ||
|
|
03c0b37086 | ||
|
|
e343035d67 | ||
|
|
f2ec975b73 | ||
|
|
7a5f937c3f | ||
|
|
49b90bc9a3 | ||
|
|
e0f33846d1 | ||
|
|
bab4f1731b | ||
|
|
ee9035b7f0 | ||
|
|
62b7bb8fc8 | ||
|
|
5798952ea8 | ||
|
|
c37affaa42 | ||
|
|
ebdd24ce64 | ||
|
|
1c09f313cc | ||
|
|
76c74ad3de | ||
|
|
b7f78d8c49 | ||
|
|
76f81ff236 | ||
|
|
543e074d60 | ||
|
|
c39d18c1da | ||
|
|
1de251d575 | ||
|
|
722aebdf11 | ||
|
|
66c688c656 | ||
|
|
5d1712e309 | ||
|
|
b2d45f1f2d | ||
|
|
2766504948 | ||
|
|
e3b45c9cbb | ||
|
|
256b1c9c28 | ||
|
|
52f26187ca | ||
|
|
e44a6189fa | ||
|
|
30a88615ba | ||
|
|
d7d19ea9ba | ||
|
|
04b141749f | ||
|
|
b616dc3d20 | ||
|
|
117bba794c | ||
|
|
1f8ba5c958 | ||
|
|
c0e6d00bd3 | ||
|
|
1485330547 | ||
|
|
8213a4a2b6 | ||
|
|
425a54019b | ||
|
|
82907ad6ca | ||
|
|
729268d065 | ||
|
|
8a5c4a4972 | ||
|
|
80bfce59ad | ||
|
|
35f3da108b | ||
|
|
c8c758e639 | ||
|
|
56db0a081a | ||
|
|
0d6ea416a7 | ||
|
|
4f9f9ec971 | ||
|
|
3bbe4481f2 | ||
|
|
cac6a51cbf | ||
|
|
7780a7d652 | ||
|
|
83b6a94bfe | ||
|
|
0f1c734626 | ||
|
|
2a61cda2cd | ||
|
|
5c93557314 | ||
|
|
7cf5ddf8ac | ||
|
|
7622d14580 | ||
|
|
bddb2cd5cc | ||
|
|
96a2749355 | ||
|
|
f9e7ca4308 | ||
|
|
f03f4383af | ||
|
|
c57ccd554a | ||
|
|
9df006b49d | ||
|
|
424bb7fe11 | ||
|
|
97d58e77c6 | ||
|
|
553a229fd3 | ||
|
|
57d025910d | ||
|
|
213dd9b470 | ||
|
|
a31fc1ac01 | ||
|
|
91d641e685 | ||
|
|
740983d69e | ||
|
|
0e649f710a | ||
|
|
772ee0f967 | ||
|
|
0c6980182c | ||
|
|
47ea1ac228 | ||
|
|
6446893e3c | ||
|
|
06c012ea60 | ||
|
|
5a6954be1b | ||
|
|
0f1ca16457 | ||
|
|
72f0a47563 | ||
|
|
751cfac7d7 | ||
|
|
5c1e6e904d | ||
|
|
04e3a68b61 | ||
|
|
c220ad1e69 | ||
|
|
c9ca213d4c | ||
|
|
4ec743583d | ||
|
|
5bb475c572 | ||
|
|
cd75202da8 | ||
|
|
cdba261913 | ||
|
|
179bcb777e | ||
|
|
802267b281 | ||
|
|
d6e335d6e2 | ||
|
|
884da0496c | ||
|
|
60f975aa20 | ||
|
|
a3d0509c92 | ||
|
|
2e51e1dab2 | ||
|
|
a187f103d1 | ||
|
|
3087da07db | ||
|
|
3ef8a9f3b2 | ||
|
|
d326affc4f | ||
|
|
d341db8861 | ||
|
|
cf5f0b8e74 | ||
|
|
0bcb64140f | ||
|
|
62d89fa60b | ||
|
|
de9399761b | ||
|
|
8797e73523 | ||
|
|
f00dc46607 | ||
|
|
1b41544f97 | ||
|
|
e9033136aa | ||
|
|
56d1ed77c1 | ||
|
|
9e2e600b6d | ||
|
|
99f132729c | ||
|
|
8c31d5e331 | ||
|
|
ef5f9f629a | ||
|
|
50cc17e8fa | ||
|
|
870b2ffe07 | ||
|
|
e4f266d927 | ||
|
|
5650b6f505 | ||
|
|
09d740db47 | ||
|
|
8c54b72af3 | ||
|
|
312fee4ddf | ||
|
|
03ad8f2490 | ||
|
|
fc666e2e48 | ||
|
|
3a9ec6f30c | ||
|
|
8603a3a1e9 | ||
|
|
bea9ca3616 | ||
|
|
32e7ba2f9d | ||
|
|
05a0cc156b | ||
|
|
96d6d27607 | ||
|
|
c7f291e4e4 |
@@ -38,13 +38,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -87,13 +87,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -18,11 +18,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@v3
|
||||
# Override language selection by uncommenting this and choosing your languages
|
||||
with:
|
||||
languages: go
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below).
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
uses: github/codeql-action/autobuild@v3
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -44,4 +44,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
uses: github/codeql-action/analyze@v3
|
||||
|
||||
@@ -16,11 +16,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v3
|
||||
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -33,31 +33,23 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
name: "docker: build foundationdb image by version"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/filer/foundationdb/**'
|
||||
- 'test/foundationdb/**'
|
||||
- 'docker/Dockerfile.foundationdb_large'
|
||||
- 'docker/filer_foundationdb.toml'
|
||||
- '.github/workflows/container_foundationdb_version.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
fdb_version:
|
||||
description: 'FoundationDB version to build (e.g. 7.4.5)'
|
||||
required: true
|
||||
default: '7.4.5'
|
||||
seaweedfs_ref:
|
||||
description: 'SeaweedFS git tag, branch, or commit to build'
|
||||
required: true
|
||||
default: 'master'
|
||||
image_tag:
|
||||
description: 'Optional Docker tag suffix (defaults to foundationdb_<fdb>_seaweedfs_<ref>)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-foundationdb-image:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install FoundationDB client libraries
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ca-certificates wget
|
||||
FDB_VERSION="${{ inputs.fdb_version || '7.4.5' }}"
|
||||
case "${FDB_VERSION}_amd64" in
|
||||
"7.4.5_amd64") EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;;
|
||||
"7.3.43_amd64") EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;;
|
||||
*)
|
||||
echo "Unsupported FoundationDB version ${FDB_VERSION} for CI client install" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
|
||||
wget --timeout=30 --tries=3 -O "${PACKAGE}" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}"
|
||||
echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c -
|
||||
sudo dpkg -i "${PACKAGE}"
|
||||
rm "${PACKAGE}"
|
||||
sudo ldconfig
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Run FoundationDB tagged tests
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
run: |
|
||||
go test ./weed/filer/foundationdb -tags foundationdb -count=1
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
env:
|
||||
FDB_VERSION_INPUT: ${{ inputs.fdb_version }}
|
||||
SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
|
||||
CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sanitize() {
|
||||
local value="$1"
|
||||
value="${value,,}"
|
||||
value="${value// /-}"
|
||||
value="${value//[^a-z0-9_.-]/-}"
|
||||
value="${value#-}"
|
||||
value="${value%-}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
version="${FDB_VERSION_INPUT}"
|
||||
seaweed="${SEAWEEDFS_REF_INPUT}"
|
||||
tag="${CUSTOM_TAG_INPUT}"
|
||||
# Use defaults for PR builds
|
||||
if [ -z "$version" ]; then
|
||||
version="7.4.5"
|
||||
fi
|
||||
if [ -z "$seaweed" ]; then
|
||||
if [ "$EVENT_NAME" = "pull_request" ]; then
|
||||
seaweed="${HEAD_REF}"
|
||||
else
|
||||
seaweed="${REF_NAME}"
|
||||
fi
|
||||
fi
|
||||
sanitized_version="$(sanitize "$version")"
|
||||
if [ -z "$sanitized_version" ]; then
|
||||
echo "Unable to sanitize FoundationDB version '$version'." >&2
|
||||
exit 1
|
||||
fi
|
||||
sanitized_seaweed="$(sanitize "$seaweed")"
|
||||
if [ -z "$sanitized_seaweed" ]; then
|
||||
echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$tag" ]; then
|
||||
tag="foundationdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
|
||||
else
|
||||
tag="$(sanitize "$tag")"
|
||||
fi
|
||||
if [ -z "$tag" ]; then
|
||||
echo "Resulting Docker tag is empty." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Determine branch to build
|
||||
id: branch
|
||||
run: |
|
||||
if [ -n "${{ inputs.seaweedfs_ref }}" ]; then
|
||||
echo "branch=${{ inputs.seaweedfs_ref }}" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
echo "branch=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.foundationdb_large
|
||||
build-args: |
|
||||
FDB_VERSION=${{ inputs.fdb_version || '7.4.5' }}
|
||||
BRANCH=${{ steps.branch.outputs.branch }}
|
||||
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
|
||||
# Currently only building for amd64. To enable ARM64, verify package availability and add checksums.
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.tag.outputs.full_image || 'seaweedfs:foundationdb-test' }}
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
@@ -10,190 +10,58 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386]
|
||||
include:
|
||||
- platform: amd64
|
||||
qemu: false
|
||||
- platform: arm64
|
||||
qemu: true
|
||||
- platform: arm
|
||||
qemu: true
|
||||
- platform: 386
|
||||
qemu: true
|
||||
|
||||
build-latest-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
# Clean Docker aggressively
|
||||
sudo docker system prune -af --volumes
|
||||
# Clean Go cache if it exists
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=latest
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.qemu
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Build ${{ matrix.platform }}
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
tags: ghcr.io/chrislusf/seaweedfs:latest-${{ matrix.platform }}
|
||||
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
run: |
|
||||
# Clean up Docker build cache and temporary files
|
||||
sudo docker system prune -f
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=latest
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Install crane
|
||||
run: |
|
||||
# Install crane for efficient multi-arch image copying
|
||||
cd $(mktemp -d)
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
- name: Create and push manifest
|
||||
run: |
|
||||
# Create manifest on GHCR first (no rate limits)
|
||||
echo "Creating GHCR manifest (no rate limits)..."
|
||||
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:latest \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-386
|
||||
|
||||
# Copy the complete multi-arch image from GHCR to Docker Hub
|
||||
# This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
|
||||
echo "Copying manifest from GHCR to Docker Hub..."
|
||||
|
||||
# Function to retry with exponential backoff for Docker Hub operations
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
# Use crane or skopeo to copy, fallback to docker if not available
|
||||
if command -v crane &> /dev/null; then
|
||||
echo "Using crane to copy..."
|
||||
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:latest chrislusf/seaweedfs:latest
|
||||
elif command -v skopeo &> /dev/null; then
|
||||
echo "Using skopeo to copy..."
|
||||
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:latest docker://chrislusf/seaweedfs:latest
|
||||
else
|
||||
echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
|
||||
# Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
|
||||
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:latest \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-386
|
||||
fi
|
||||
|
||||
@@ -1,152 +1,57 @@
|
||||
name: "docker: build release containers for normal volume"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386]
|
||||
include:
|
||||
- platform: amd64
|
||||
qemu: false
|
||||
- platform: arm64
|
||||
qemu: true
|
||||
- platform: arm
|
||||
qemu: true
|
||||
- platform: 386
|
||||
qemu: true
|
||||
build-default-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
# Clean Docker aggressively
|
||||
sudo docker system prune -af --volumes
|
||||
# Clean Go cache if it exists
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag
|
||||
flavor: latest=false
|
||||
- name: Set up QEMU
|
||||
if: matrix.qemu
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build ${{ matrix.platform }}
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
|
||||
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
run: |
|
||||
# Clean up Docker build cache and temporary files
|
||||
sudo docker system prune -f
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag
|
||||
flavor: latest=false
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Create and push manifest
|
||||
run: |
|
||||
# Function to retry command with exponential backoff
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
# Create manifest with retry
|
||||
retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
|
||||
${{ steps.docker_meta.outputs.tags }}-amd64 \
|
||||
${{ steps.docker_meta.outputs.tags }}-arm64 \
|
||||
${{ steps.docker_meta.outputs.tags }}-arm \
|
||||
${{ steps.docker_meta.outputs.tags }}-386
|
||||
|
||||
@@ -1,153 +1,59 @@
|
||||
name: "docker: build release containers for large volume"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386]
|
||||
include:
|
||||
- platform: amd64
|
||||
qemu: false
|
||||
- platform: arm64
|
||||
qemu: true
|
||||
- platform: arm
|
||||
qemu: true
|
||||
- platform: 386
|
||||
qemu: true
|
||||
|
||||
build-large-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
# Clean Docker aggressively
|
||||
sudo docker system prune -af --volumes
|
||||
# Clean Go cache if it exists
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=_large_disk
|
||||
flavor: latest=false
|
||||
- name: Set up QEMU
|
||||
if: matrix.qemu
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_large_disk
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build ${{ matrix.platform }}
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: |
|
||||
TAGS=5BytesOffset
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
|
||||
build-args: TAGS=5BytesOffset
|
||||
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
run: |
|
||||
# Clean up Docker build cache and temporary files
|
||||
sudo docker system prune -f
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=_large_disk
|
||||
flavor: latest=false
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Create and push manifest
|
||||
run: |
|
||||
# Function to retry command with exponential backoff
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
# Create manifest with retry
|
||||
retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
|
||||
${{ steps.docker_meta.outputs.tags }}-amd64 \
|
||||
${{ steps.docker_meta.outputs.tags }}-arm64 \
|
||||
${{ steps.docker_meta.outputs.tags }}-arm \
|
||||
${{ steps.docker_meta.outputs.tags }}-386
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
name: "docker: build release containers for rocksdb"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -20,54 +17,42 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Free Disk Space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
|
||||
sudo docker system prune -af
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=_large_disk_rocksdb
|
||||
flavor: latest=false
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_large_disk_rocksdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.rocksdb_large
|
||||
build-args: |
|
||||
BRANCH=${{ github.sha }}
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
name: "docker: build release containers for all tags"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -19,23 +16,11 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Free Disk Space
|
||||
run: |
|
||||
echo "Before cleanup:"
|
||||
df -h
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /usr/local/lib/android
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
||||
sudo docker system prune -af
|
||||
echo "After cleanup:"
|
||||
df -h
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -47,19 +32,22 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
@@ -68,5 +56,3 @@ jobs:
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
name: "docker: build release containers for all tags and large volume"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -19,37 +16,38 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Free Disk Space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
|
||||
sudo docker system prune -af
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=_large_disk_full
|
||||
flavor: latest=false
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_large_disk_full
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
@@ -58,5 +56,3 @@ jobs:
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
name: "docker: build release containers for foundationdb"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_foundationdb:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_large_disk_foundationdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Determine branch to build
|
||||
id: branch
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "push" ] && [ -n "${{ github.ref_name }}" ]; then
|
||||
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "branch=master" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.foundationdb_large
|
||||
build-args: |
|
||||
BRANCH=${{ steps.branch.outputs.branch }}
|
||||
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
|
||||
@@ -1,245 +0,0 @@
|
||||
name: "docker: build all release containers (unified)"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Limit concurrent builds to avoid rate limits
|
||||
concurrency:
|
||||
group: release-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# Build sequentially to avoid rate limits
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
include:
|
||||
# Normal volume - multi-arch
|
||||
- variant: normal
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
|
||||
# Large disk - multi-arch
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
|
||||
# Full tags - amd64 only
|
||||
- variant: full
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
|
||||
# Large disk + full tags - amd64 only
|
||||
- variant: large_disk_full
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
sudo docker system prune -af --volumes
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=${{ matrix.tag_suffix }}
|
||||
flavor: latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
if: contains(matrix.platforms, 'arm')
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build and push ${{ matrix.variant }}
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: ${{ matrix.platforms }}
|
||||
# Push to GHCR to avoid Docker Hub rate limits on pulls
|
||||
tags: |
|
||||
ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
|
||||
build-args: |
|
||||
${{ matrix.build_args }}
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
run: |
|
||||
sudo docker system prune -f
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
copy-to-dockerhub:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
matrix:
|
||||
variant: [normal, large_disk, full, large_disk_full, rocksdb]
|
||||
include:
|
||||
- variant: normal
|
||||
tag_suffix: ""
|
||||
- variant: large_disk
|
||||
tag_suffix: _large_disk
|
||||
- variant: full
|
||||
tag_suffix: _full
|
||||
- variant: large_disk_full
|
||||
tag_suffix: _large_disk_full
|
||||
- variant: rocksdb
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
|
||||
run: |
|
||||
# Function to retry with exponential backoff
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
# Copy multi-arch image from GHCR to Docker Hub with retry
|
||||
# This is much more efficient than pulling/pushing individual arch images
|
||||
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
|
||||
retry_with_backoff crane copy \
|
||||
ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }} \
|
||||
chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
|
||||
|
||||
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [copy-to-dockerhub]
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: "3.18.4"
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
name: "docker: build rocksdb image by version"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag or branch to build (e.g. v10.5.1)'
|
||||
required: true
|
||||
default: 'v10.5.1'
|
||||
seaweedfs_ref:
|
||||
description: 'SeaweedFS git tag, branch, or commit to build'
|
||||
required: true
|
||||
default: 'master'
|
||||
image_tag:
|
||||
description: 'Optional Docker tag suffix (defaults to rocksdb_<rocksdb>_seaweedfs_<ref>)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-rocksdb-image:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
env:
|
||||
ROCKSDB_VERSION_INPUT: ${{ inputs.rocksdb_version }}
|
||||
SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
|
||||
CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sanitize() {
|
||||
local value="$1"
|
||||
value="${value,,}"
|
||||
value="${value// /-}"
|
||||
value="${value//[^a-z0-9_.-]/-}"
|
||||
value="${value#-}"
|
||||
value="${value%-}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
version="${ROCKSDB_VERSION_INPUT}"
|
||||
seaweed="${SEAWEEDFS_REF_INPUT}"
|
||||
tag="${CUSTOM_TAG_INPUT}"
|
||||
if [ -z "$version" ]; then
|
||||
echo "RocksDB version input is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$seaweed" ]; then
|
||||
echo "SeaweedFS ref input is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
sanitized_version="$(sanitize "$version")"
|
||||
if [ -z "$sanitized_version" ]; then
|
||||
echo "Unable to sanitize RocksDB version '$version'." >&2
|
||||
exit 1
|
||||
fi
|
||||
sanitized_seaweed="$(sanitize "$seaweed")"
|
||||
if [ -z "$sanitized_seaweed" ]; then
|
||||
echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$tag" ]; then
|
||||
tag="rocksdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
|
||||
fi
|
||||
tag="${tag,,}"
|
||||
tag="${tag// /-}"
|
||||
tag="${tag//[^a-z0-9_.-]/-}"
|
||||
tag="${tag#-}"
|
||||
tag="${tag%-}"
|
||||
if [ -z "$tag" ]; then
|
||||
echo "Resulting Docker tag is empty." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
file: ./docker/Dockerfile.rocksdb_large
|
||||
build-args: |
|
||||
ROCKSDB_VERSION=${{ inputs.rocksdb_version }}
|
||||
BRANCH=${{ inputs.seaweedfs_ref }}
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.tag.outputs.full_image }}
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
@@ -21,10 +21,10 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.24'
|
||||
|
||||
|
||||
@@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261
|
||||
uses: actions/dependency-review-action@bc41886e18ea39df68b1b1245f4184881938e050
|
||||
|
||||
@@ -24,62 +24,22 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v2
|
||||
uses: actions/setup-go@8e57b58e57be52ac95949151e2777ffda8501267 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-e2e-
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# Use faster mirrors and install with timeout
|
||||
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse" | sudo tee /etc/apt/sources.list
|
||||
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
|
||||
|
||||
sudo apt-get update --fix-missing
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends fuse
|
||||
|
||||
# Verify FUSE installation
|
||||
echo "FUSE version: $(fusermount --version 2>&1 || echo 'fusermount not found')"
|
||||
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse
|
||||
|
||||
- name: Start SeaweedFS
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
# Enable Docker buildkit for better caching
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
# Build with retry logic
|
||||
for i in {1..3}; do
|
||||
echo "Build attempt $i/3"
|
||||
if make build_e2e; then
|
||||
echo "Build successful on attempt $i"
|
||||
break
|
||||
elif [ $i -eq 3 ]; then
|
||||
echo "Build failed after 3 attempts"
|
||||
exit 1
|
||||
else
|
||||
echo "Build attempt $i failed, retrying in 30 seconds..."
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
# Start services with wait
|
||||
docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
timeout-minutes: 5
|
||||
run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Run FIO 4k
|
||||
timeout-minutes: 15
|
||||
@@ -134,7 +94,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
name: "EC Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
ec-integration-tests:
|
||||
name: EC Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run EC Integration Tests
|
||||
working-directory: test/erasure_coding
|
||||
run: |
|
||||
go test -v
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/ec-test-logs
|
||||
# Find all temp directories created by the tests (they persist on failure with t.TempDir())
|
||||
find /tmp -maxdepth 1 -type d \( -name "TestEC*" -o -name "TestDisk*" -o -name "TestCross*" -o -name "TestEvacuation*" \) 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
# Copy the entire directory structure to preserve organization
|
||||
cp -r "$dir" /tmp/ec-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
# List what we collected
|
||||
echo "Collected logs:"
|
||||
find /tmp/ec-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: |
|
||||
/tmp/ec-test-logs/
|
||||
test/erasure_coding/
|
||||
if-no-files-found: warn
|
||||
@@ -22,7 +22,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
GO_VERSION: '1.21'
|
||||
TEST_TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
@@ -33,10 +33,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: fuse-integration-test-results
|
||||
path: |
|
||||
|
||||
@@ -21,27 +21,18 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v2
|
||||
uses: actions/setup-go@8e57b58e57be52ac95949151e2777ffda8501267 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd weed; go get -v -t -d ./...
|
||||
|
||||
- name: Go Vet (excluding protobuf lock copying)
|
||||
run: |
|
||||
cd weed
|
||||
# Run go vet and filter out known protobuf MessageState lock copying warnings
|
||||
# These are expected in generated protobuf code with embedded sync.Mutex and are safe in practice
|
||||
go vet -v ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-output.txt
|
||||
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
|
||||
if grep -q "vet:" vet-output.txt; then exit 1; fi
|
||||
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: "helm: publish charts"
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: v3.18.4
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -25,13 +25,13 @@ jobs:
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.9'
|
||||
check-latest: true
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: helm/chart-testing-action@v2.8.0
|
||||
uses: helm/chart-testing-action@v2.7.0
|
||||
|
||||
- name: Run chart-testing (list-changed)
|
||||
id: list-changed
|
||||
@@ -44,82 +44,8 @@ jobs:
|
||||
- name: Run chart-testing (lint)
|
||||
run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
|
||||
|
||||
- name: Verify template rendering
|
||||
run: |
|
||||
set -e
|
||||
CHART_DIR="k8s/charts/seaweedfs"
|
||||
|
||||
echo "=== Testing default configuration ==="
|
||||
helm template test $CHART_DIR > /tmp/default.yaml
|
||||
echo "✓ Default configuration renders successfully"
|
||||
|
||||
echo "=== Testing with S3 enabled ==="
|
||||
helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
|
||||
grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
|
||||
echo "✓ S3 deployment renders correctly"
|
||||
|
||||
echo "=== Testing with all-in-one mode ==="
|
||||
helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
|
||||
grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
|
||||
echo "✓ All-in-one deployment renders correctly"
|
||||
|
||||
echo "=== Testing with security enabled ==="
|
||||
helm template test $CHART_DIR --set global.enableSecurity=true > /tmp/security.yaml
|
||||
grep -q "security-config" /tmp/security.yaml
|
||||
echo "✓ Security configuration renders correctly"
|
||||
|
||||
echo "=== Testing with monitoring enabled ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set global.monitoring.enabled=true \
|
||||
--set global.monitoring.gatewayHost=prometheus \
|
||||
--set global.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
|
||||
echo "✓ Monitoring configuration renders correctly"
|
||||
|
||||
echo "=== Testing with PVC storage ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set master.data.type=persistentVolumeClaim \
|
||||
--set master.data.size=10Gi \
|
||||
--set master.data.storageClass=standard > /tmp/pvc.yaml
|
||||
grep -q "PersistentVolumeClaim" /tmp/pvc.yaml
|
||||
echo "✓ PVC configuration renders correctly"
|
||||
|
||||
echo "=== Testing with custom replicas ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set master.replicas=3 \
|
||||
--set filer.replicas=2 \
|
||||
--set volume.replicas=3 > /tmp/replicas.yaml
|
||||
echo "✓ Custom replicas configuration renders correctly"
|
||||
|
||||
echo "=== Testing filer with S3 gateway ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set filer.s3.enabled=true \
|
||||
--set filer.s3.enableAuth=true > /tmp/filer-s3.yaml
|
||||
echo "✓ Filer S3 gateway renders correctly"
|
||||
|
||||
echo "=== Testing SFTP enabled ==="
|
||||
helm template test $CHART_DIR --set sftp.enabled=true > /tmp/sftp.yaml
|
||||
grep -q "seaweedfs-sftp" /tmp/sftp.yaml
|
||||
echo "✓ SFTP deployment renders correctly"
|
||||
|
||||
echo "=== Testing ingress configurations ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set master.ingress.enabled=true \
|
||||
--set filer.ingress.enabled=true \
|
||||
--set s3.enabled=true \
|
||||
--set s3.ingress.enabled=true > /tmp/ingress.yaml
|
||||
grep -q "kind: Ingress" /tmp/ingress.yaml
|
||||
echo "✓ Ingress configurations render correctly"
|
||||
|
||||
echo "=== Testing COSI driver ==="
|
||||
helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
|
||||
grep -q "seaweedfs-cosi" /tmp/cosi.yaml
|
||||
echo "✓ COSI driver renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "✅ All template rendering tests passed!"
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.13.0
|
||||
uses: helm/kind-action@v1.12.0
|
||||
|
||||
- name: Run chart-testing (install)
|
||||
run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
name: Java Client Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'other/java/**'
|
||||
- 'weed/**'
|
||||
- '.github/workflows/java_integration_tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'other/java/**'
|
||||
- 'weed/**'
|
||||
- '.github/workflows/java_integration_tests.yml'
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
java: ['11', '17']
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go install -buildvcs=false
|
||||
weed version
|
||||
|
||||
- name: Start SeaweedFS Server
|
||||
run: |
|
||||
# Create clean data directory
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)"
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS with optimized settings for CI
|
||||
# Include S3 API for s3copier integration tests
|
||||
weed server -dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.electionTimeout=1s \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
-volume.max=100 \
|
||||
-volume.preStopSeconds=1 \
|
||||
-master.peers=none \
|
||||
-filer -filer.maxMB=64 \
|
||||
-s3 -s3.port=8333 \
|
||||
-s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
-master.port=9333 \
|
||||
-volume.port=8080 \
|
||||
-filer.port=8888 \
|
||||
-metricsPort=9324 > seaweedfs.log 2>&1 &
|
||||
|
||||
SERVER_PID=$!
|
||||
echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV
|
||||
echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV
|
||||
echo "SeaweedFS server started with PID: $SERVER_PID"
|
||||
|
||||
- name: Wait for SeaweedFS Components
|
||||
run: |
|
||||
echo "Waiting for SeaweedFS components to start..."
|
||||
|
||||
# Wait for master
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
|
||||
echo "✓ Master server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for master server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Wait for volume
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
|
||||
echo "✓ Volume server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for volume server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Wait for filer
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
|
||||
echo "✓ Filer is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for filer... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Wait for S3 API
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
|
||||
echo "✓ S3 API is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 API... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "✓ All SeaweedFS components are ready!"
|
||||
|
||||
# Display cluster status
|
||||
echo "Cluster status:"
|
||||
curl -s http://localhost:9333/cluster/status | head -20
|
||||
|
||||
- name: Build and Install SeaweedFS Client
|
||||
working-directory: other/java/client
|
||||
run: |
|
||||
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Run Client Unit Tests
|
||||
working-directory: other/java/client
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
|
||||
|
||||
- name: Run Client Integration Tests
|
||||
working-directory: other/java/client
|
||||
env:
|
||||
SEAWEEDFS_TEST_ENABLED: true
|
||||
run: |
|
||||
mvn test -Dtest=*IntegrationTest
|
||||
|
||||
- name: Run HDFS3 Configuration Tests
|
||||
working-directory: other/java/hdfs3
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Run S3 ETag Validation Tests (Issue #7768)
|
||||
working-directory: other/java/s3copier
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
S3_ACCESS_KEY: some_access_key1
|
||||
S3_SECRET_KEY: some_secret_key1
|
||||
run: |
|
||||
echo "Running S3 ETag validation tests against $S3_ENDPOINT"
|
||||
mvn test -Dtest=ETagValidationTest \
|
||||
-DS3_ENDPOINT=$S3_ENDPOINT \
|
||||
-DS3_ACCESS_KEY=$S3_ACCESS_KEY \
|
||||
-DS3_SECRET_KEY=$S3_SECRET_KEY \
|
||||
-Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Display logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== SeaweedFS Server Log ==="
|
||||
tail -100 seaweedfs.log || echo "No server log"
|
||||
echo ""
|
||||
echo "=== Cluster Status ==="
|
||||
curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster"
|
||||
echo ""
|
||||
echo "=== Process Status ==="
|
||||
ps aux | grep weed || echo "No weed processes"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
# Stop server using stored PID
|
||||
if [ -n "$SERVER_PID" ]; then
|
||||
echo "Stopping SeaweedFS server (PID: $SERVER_PID)"
|
||||
kill -9 $SERVER_PID 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Fallback: kill any remaining weed processes
|
||||
pkill -f "weed server" || true
|
||||
|
||||
# Clean up data directory
|
||||
if [ -n "$WEED_DATA_DIR" ]; then
|
||||
echo "Cleaning up data directory: $WEED_DATA_DIR"
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
fi
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
name: Java Client Unit Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'other/java/**'
|
||||
- '.github/workflows/java_unit_tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'other/java/**'
|
||||
- '.github/workflows/java_unit_tests.yml'
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
java: ['8', '11', '17', '21']
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
|
||||
- name: Build and Install SeaweedFS Client
|
||||
working-directory: other/java/client
|
||||
run: |
|
||||
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Run Client Unit Tests
|
||||
working-directory: other/java/client
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
|
||||
|
||||
- name: Run HDFS3 Configuration Tests
|
||||
working-directory: other/java/hdfs3
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: test-reports-java-${{ matrix.java }}
|
||||
path: |
|
||||
other/java/client/target/surefire-reports/
|
||||
other/java/hdfs3/target/surefire-reports/
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
name: "Kafka Quick Test (Load Test with Schema Registry)"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/kafka-quicktest
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
kafka-client-quicktest:
|
||||
name: Kafka Client Load Test (Quick)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# Ensure make is available
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y make
|
||||
|
||||
- name: Validate test setup
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
make validate-setup
|
||||
|
||||
- name: Run quick-test
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
# Run the quick-test target which includes:
|
||||
# 1. Building the gateway
|
||||
# 2. Starting all services (SeaweedFS, MQ broker, Schema Registry)
|
||||
# 3. Registering Avro schemas
|
||||
# 4. Running a 1-minute load test with Avro messages
|
||||
# Override GOARCH to build for AMD64 (GitHub Actions runners are x86_64)
|
||||
GOARCH=amd64 make quick-test
|
||||
env:
|
||||
# Docker Compose settings
|
||||
COMPOSE_HTTP_TIMEOUT: 300
|
||||
DOCKER_CLIENT_TIMEOUT: 300
|
||||
# Test parameters (set by quick-test, but can override)
|
||||
TEST_DURATION: 60s
|
||||
PRODUCER_COUNT: 1
|
||||
CONSUMER_COUNT: 1
|
||||
MESSAGE_RATE: 10
|
||||
VALUE_TYPE: avro
|
||||
|
||||
- name: Show test results
|
||||
if: always()
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
echo "========================================="
|
||||
echo "Test Results"
|
||||
echo "========================================="
|
||||
make show-results || echo "Could not retrieve results"
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
echo "========================================="
|
||||
echo "Service Logs"
|
||||
echo "========================================="
|
||||
|
||||
echo "Checking running containers..."
|
||||
docker compose ps || true
|
||||
|
||||
echo "========================================="
|
||||
echo "Master Logs"
|
||||
echo "========================================="
|
||||
docker compose logs --tail=100 seaweedfs-master 2>&1 || echo "No master logs available"
|
||||
|
||||
echo "========================================="
|
||||
echo "MQ Broker Logs (Last 100 lines)"
|
||||
echo "========================================="
|
||||
docker compose logs --tail=100 seaweedfs-mq-broker 2>&1 || echo "No broker logs available"
|
||||
|
||||
echo "========================================="
|
||||
echo "Kafka Gateway Logs (FULL - Critical for debugging)"
|
||||
echo "========================================="
|
||||
docker compose logs kafka-gateway 2>&1 || echo "ERROR: Could not retrieve kafka-gateway logs"
|
||||
|
||||
echo "========================================="
|
||||
echo "Schema Registry Logs (FULL)"
|
||||
echo "========================================="
|
||||
docker compose logs schema-registry 2>&1 || echo "ERROR: Could not retrieve schema-registry logs"
|
||||
|
||||
echo "========================================="
|
||||
echo "Load Test Logs"
|
||||
echo "========================================="
|
||||
docker compose logs --tail=100 kafka-client-loadtest 2>&1 || echo "No loadtest logs available"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
# Stop containers first
|
||||
docker compose --profile loadtest --profile monitoring down -v --remove-orphans || true
|
||||
# Clean up data with sudo to handle Docker root-owned files
|
||||
sudo rm -rf data/* || true
|
||||
# Clean up binary
|
||||
rm -f weed-linux-* || true
|
||||
@@ -1,825 +0,0 @@
|
||||
name: "Kafka Gateway Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/kafka-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
# Force different runners for better isolation
|
||||
env:
|
||||
FORCE_RUNNER_SEPARATION: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
kafka-unit-tests:
|
||||
name: Kafka Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [unit-tests-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 1g --hostname kafka-unit-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Container Environment
|
||||
run: |
|
||||
apk add --no-cache git
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Kafka Gateway Unit Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Set process limits for container isolation
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
go test -v -timeout 10s ./unit/...
|
||||
|
||||
kafka-integration-tests:
|
||||
name: Kafka Integration Tests (Critical)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [integration-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_TEST_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Integration Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps
|
||||
ulimit -n 2048 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Higher limits for integration tests
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 200 || echo "Warning: Could not set process limit"
|
||||
go test -v -timeout 90s ./integration/...
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
|
||||
kafka-e2e-tests:
|
||||
name: Kafka End-to-End Tests (with SMQ)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [e2e-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 2.0 --memory 2g --hostname kafka-e2e-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_E2E_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Setup E2E Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps curl netcat-openbsd
|
||||
ulimit -n 2048 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Warm Go module cache
|
||||
run: |
|
||||
# Warm cache for root module
|
||||
go mod download || true
|
||||
# Warm cache for kafka test module
|
||||
cd test/kafka
|
||||
go mod download || true
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Use go mod download with timeout to prevent hanging
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
set -e
|
||||
cd $GITHUB_WORKSPACE
|
||||
# Build weed binary
|
||||
go build -o /usr/local/bin/weed ./weed
|
||||
# Start SeaweedFS components with MQ brokers
|
||||
export WEED_DATA_DIR=/tmp/seaweedfs-e2e-$RANDOM
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
|
||||
nohup weed -v 1 server \
|
||||
-ip="127.0.0.1" \
|
||||
-ip.bind="0.0.0.0" \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.port=9333 \
|
||||
-volume.port=8081 \
|
||||
-filer.port=8888 \
|
||||
-filer=true \
|
||||
-metricsPort=9325 \
|
||||
-master.peers=none \
|
||||
> /tmp/weed-server.log 2>&1 &
|
||||
|
||||
# Wait for master to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
|
||||
echo "SeaweedFS master HTTP is up"; break
|
||||
fi
|
||||
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Wait for master gRPC to be ready (this is what broker discovery uses)
|
||||
echo "Waiting for master gRPC port..."
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z 127.0.0.1 19333; then
|
||||
echo "✓ SeaweedFS master gRPC is up (port 19333)"
|
||||
break
|
||||
fi
|
||||
echo " Waiting for master gRPC... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Give server time to initialize all components including gRPC services
|
||||
echo "Waiting for SeaweedFS components to initialize..."
|
||||
sleep 15
|
||||
|
||||
# Additional wait specifically for gRPC services to be ready for streaming
|
||||
echo "Allowing extra time for master gRPC streaming services to initialize..."
|
||||
sleep 10
|
||||
|
||||
# Start MQ broker with maximum verbosity for debugging
|
||||
echo "Starting MQ broker..."
|
||||
nohup weed -v 3 mq.broker \
|
||||
-master="127.0.0.1:9333" \
|
||||
-ip="127.0.0.1" \
|
||||
-port=17777 \
|
||||
-logFlushInterval=0 \
|
||||
> /tmp/weed-mq-broker.log 2>&1 &
|
||||
|
||||
# Wait for broker to be ready with better error reporting
|
||||
sleep 15
|
||||
broker_ready=false
|
||||
for i in $(seq 1 20); do
|
||||
if nc -z 127.0.0.1 17777; then
|
||||
echo "SeaweedFS MQ broker is up"
|
||||
broker_ready=true
|
||||
break
|
||||
fi
|
||||
echo "Waiting for MQ broker... ($i/20)"; sleep 1
|
||||
done
|
||||
|
||||
# Give broker additional time to register with master
|
||||
if [ "$broker_ready" = true ]; then
|
||||
echo "Allowing broker to register with master..."
|
||||
sleep 30
|
||||
|
||||
# Check if broker is properly registered by querying cluster nodes
|
||||
echo "Cluster status after broker registration:"
|
||||
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
|
||||
|
||||
echo "Checking cluster topology (includes registered components):"
|
||||
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
|
||||
|
||||
echo "Verifying broker discovery via master client debug:"
|
||||
echo "If broker registration is successful, it should appear in dir status"
|
||||
|
||||
echo "Testing gRPC connectivity with weed binary:"
|
||||
echo "This simulates what the gateway does during broker discovery..."
|
||||
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
|
||||
echo "Shell test results:"
|
||||
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
|
||||
fi
|
||||
|
||||
# Check if broker failed to start and show logs
|
||||
if [ "$broker_ready" = false ]; then
|
||||
echo "ERROR: MQ broker failed to start. Broker logs:"
|
||||
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
|
||||
echo "Server logs:"
|
||||
tail -20 /tmp/weed-server.log || echo "No server logs found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run End-to-End Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Higher limits for E2E tests
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 200 || echo "Warning: Could not set process limit"
|
||||
|
||||
# Allow additional time for all background processes to settle
|
||||
echo "Allowing additional settlement time for SeaweedFS ecosystem..."
|
||||
sleep 15
|
||||
|
||||
# Run tests and capture result
|
||||
if ! go test -v -timeout 180s ./e2e/...; then
|
||||
echo "========================================="
|
||||
echo "Tests failed! Showing debug information:"
|
||||
echo "========================================="
|
||||
echo "Server logs (last 50 lines):"
|
||||
tail -50 /tmp/weed-server.log || echo "No server logs"
|
||||
echo "========================================="
|
||||
echo "Broker logs (last 50 lines):"
|
||||
tail -50 /tmp/weed-mq-broker.log || echo "No broker logs"
|
||||
echo "========================================="
|
||||
exit 1
|
||||
fi
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
kafka-consumer-group-tests:
|
||||
name: Kafka Consumer Group Tests (Highly Isolated)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [consumer-group-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 2g --ulimit nofile=512:512 --hostname kafka-consumer-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_CONSUMER_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Setup Consumer Group Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps curl netcat-openbsd
|
||||
ulimit -n 256 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Warm Go module cache
|
||||
run: |
|
||||
# Warm cache for root module
|
||||
go mod download || true
|
||||
# Warm cache for kafka test module
|
||||
cd test/kafka
|
||||
go mod download || true
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Use go mod download with timeout to prevent hanging
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
set -e
|
||||
cd $GITHUB_WORKSPACE
|
||||
# Build weed binary
|
||||
go build -o /usr/local/bin/weed ./weed
|
||||
# Start SeaweedFS components with MQ brokers
|
||||
export WEED_DATA_DIR=/tmp/seaweedfs-mq-$RANDOM
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
|
||||
nohup weed -v 1 server \
|
||||
-ip="127.0.0.1" \
|
||||
-ip.bind="0.0.0.0" \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.port=9333 \
|
||||
-volume.port=8081 \
|
||||
-filer.port=8888 \
|
||||
-filer=true \
|
||||
-metricsPort=9325 \
|
||||
-master.peers=none \
|
||||
> /tmp/weed-server.log 2>&1 &
|
||||
|
||||
# Wait for master to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
|
||||
echo "SeaweedFS master HTTP is up"; break
|
||||
fi
|
||||
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Wait for master gRPC to be ready (this is what broker discovery uses)
|
||||
echo "Waiting for master gRPC port..."
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z 127.0.0.1 19333; then
|
||||
echo "✓ SeaweedFS master gRPC is up (port 19333)"
|
||||
break
|
||||
fi
|
||||
echo " Waiting for master gRPC... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Give server time to initialize all components including gRPC services
|
||||
echo "Waiting for SeaweedFS components to initialize..."
|
||||
sleep 15
|
||||
|
||||
# Additional wait specifically for gRPC services to be ready for streaming
|
||||
echo "Allowing extra time for master gRPC streaming services to initialize..."
|
||||
sleep 10
|
||||
|
||||
# Start MQ broker with maximum verbosity for debugging
|
||||
echo "Starting MQ broker..."
|
||||
nohup weed -v 3 mq.broker \
|
||||
-master="127.0.0.1:9333" \
|
||||
-ip="127.0.0.1" \
|
||||
-port=17777 \
|
||||
-logFlushInterval=0 \
|
||||
> /tmp/weed-mq-broker.log 2>&1 &
|
||||
|
||||
# Wait for broker to be ready with better error reporting
|
||||
sleep 15
|
||||
broker_ready=false
|
||||
for i in $(seq 1 20); do
|
||||
if nc -z 127.0.0.1 17777; then
|
||||
echo "SeaweedFS MQ broker is up"
|
||||
broker_ready=true
|
||||
break
|
||||
fi
|
||||
echo "Waiting for MQ broker... ($i/20)"; sleep 1
|
||||
done
|
||||
|
||||
# Give broker additional time to register with master
|
||||
if [ "$broker_ready" = true ]; then
|
||||
echo "Allowing broker to register with master..."
|
||||
sleep 30
|
||||
|
||||
# Check if broker is properly registered by querying cluster nodes
|
||||
echo "Cluster status after broker registration:"
|
||||
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
|
||||
|
||||
echo "Checking cluster topology (includes registered components):"
|
||||
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
|
||||
|
||||
echo "Verifying broker discovery via master client debug:"
|
||||
echo "If broker registration is successful, it should appear in dir status"
|
||||
|
||||
echo "Testing gRPC connectivity with weed binary:"
|
||||
echo "This simulates what the gateway does during broker discovery..."
|
||||
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
|
||||
echo "Shell test results:"
|
||||
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
|
||||
fi
|
||||
|
||||
# Check if broker failed to start and show logs
|
||||
if [ "$broker_ready" = false ]; then
|
||||
echo "ERROR: MQ broker failed to start. Broker logs:"
|
||||
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
|
||||
echo "Server logs:"
|
||||
tail -20 /tmp/weed-server.log || echo "No server logs found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run Consumer Group Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Test consumer group functionality with explicit timeout
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
kafka-client-compatibility:
|
||||
name: Kafka Client Compatibility (with SMQ)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [client-compat-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_CLIENT_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Setup Client Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps curl netcat-openbsd
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Warm Go module cache
|
||||
run: |
|
||||
# Warm cache for root module
|
||||
go mod download || true
|
||||
# Warm cache for kafka test module
|
||||
cd test/kafka
|
||||
go mod download || true
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
set -e
|
||||
cd $GITHUB_WORKSPACE
|
||||
# Build weed binary
|
||||
go build -o /usr/local/bin/weed ./weed
|
||||
# Start SeaweedFS components with MQ brokers
|
||||
export WEED_DATA_DIR=/tmp/seaweedfs-client-$RANDOM
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
|
||||
nohup weed -v 1 server \
|
||||
-ip="127.0.0.1" \
|
||||
-ip.bind="0.0.0.0" \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.port=9333 \
|
||||
-volume.port=8081 \
|
||||
-filer.port=8888 \
|
||||
-filer=true \
|
||||
-metricsPort=9325 \
|
||||
-master.peers=none \
|
||||
> /tmp/weed-server.log 2>&1 &
|
||||
|
||||
# Wait for master to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
|
||||
echo "SeaweedFS master HTTP is up"; break
|
||||
fi
|
||||
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Wait for master gRPC to be ready (this is what broker discovery uses)
|
||||
echo "Waiting for master gRPC port..."
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z 127.0.0.1 19333; then
|
||||
echo "✓ SeaweedFS master gRPC is up (port 19333)"
|
||||
break
|
||||
fi
|
||||
echo " Waiting for master gRPC... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Give server time to initialize all components including gRPC services
|
||||
echo "Waiting for SeaweedFS components to initialize..."
|
||||
sleep 15
|
||||
|
||||
# Additional wait specifically for gRPC services to be ready for streaming
|
||||
echo "Allowing extra time for master gRPC streaming services to initialize..."
|
||||
sleep 10
|
||||
|
||||
# Start MQ broker with maximum verbosity for debugging
|
||||
echo "Starting MQ broker..."
|
||||
nohup weed -v 3 mq.broker \
|
||||
-master="127.0.0.1:9333" \
|
||||
-ip="127.0.0.1" \
|
||||
-port=17777 \
|
||||
-logFlushInterval=0 \
|
||||
> /tmp/weed-mq-broker.log 2>&1 &
|
||||
|
||||
# Wait for broker to be ready with better error reporting
|
||||
sleep 15
|
||||
broker_ready=false
|
||||
for i in $(seq 1 20); do
|
||||
if nc -z 127.0.0.1 17777; then
|
||||
echo "SeaweedFS MQ broker is up"
|
||||
broker_ready=true
|
||||
break
|
||||
fi
|
||||
echo "Waiting for MQ broker... ($i/20)"; sleep 1
|
||||
done
|
||||
|
||||
# Give broker additional time to register with master
|
||||
if [ "$broker_ready" = true ]; then
|
||||
echo "Allowing broker to register with master..."
|
||||
sleep 30
|
||||
|
||||
# Check if broker is properly registered by querying cluster nodes
|
||||
echo "Cluster status after broker registration:"
|
||||
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
|
||||
|
||||
echo "Checking cluster topology (includes registered components):"
|
||||
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
|
||||
|
||||
echo "Verifying broker discovery via master client debug:"
|
||||
echo "If broker registration is successful, it should appear in dir status"
|
||||
|
||||
echo "Testing gRPC connectivity with weed binary:"
|
||||
echo "This simulates what the gateway does during broker discovery..."
|
||||
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
|
||||
echo "Shell test results:"
|
||||
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
|
||||
fi
|
||||
|
||||
# Check if broker failed to start and show logs
|
||||
if [ "$broker_ready" = false ]; then
|
||||
echo "ERROR: MQ broker failed to start. Broker logs:"
|
||||
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
|
||||
echo "Server logs:"
|
||||
tail -20 /tmp/weed-server.log || echo "No server logs found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run Client Compatibility Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
go test -v -run "^TestClientCompatibility" -timeout 180s ./integration/...
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
kafka-smq-integration-tests:
|
||||
name: Kafka SMQ Integration Tests (Full Stack)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [smq-integration-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 2g --hostname kafka-smq-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_SMQ_INTEGRATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Setup SMQ Integration Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps curl netcat-openbsd
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Warm Go module cache
|
||||
run: |
|
||||
# Warm cache for root module
|
||||
go mod download || true
|
||||
# Warm cache for kafka test module
|
||||
cd test/kafka
|
||||
go mod download || true
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
set -e
|
||||
cd $GITHUB_WORKSPACE
|
||||
# Build weed binary
|
||||
go build -o /usr/local/bin/weed ./weed
|
||||
# Start SeaweedFS components with MQ brokers
|
||||
export WEED_DATA_DIR=/tmp/seaweedfs-smq-$RANDOM
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
|
||||
nohup weed -v 1 server \
|
||||
-ip="127.0.0.1" \
|
||||
-ip.bind="0.0.0.0" \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.port=9333 \
|
||||
-volume.port=8081 \
|
||||
-filer.port=8888 \
|
||||
-filer=true \
|
||||
-metricsPort=9325 \
|
||||
-master.peers=none \
|
||||
> /tmp/weed-server.log 2>&1 &
|
||||
|
||||
# Wait for master to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
|
||||
echo "SeaweedFS master HTTP is up"; break
|
||||
fi
|
||||
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Wait for master gRPC to be ready (this is what broker discovery uses)
|
||||
echo "Waiting for master gRPC port..."
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z 127.0.0.1 19333; then
|
||||
echo "✓ SeaweedFS master gRPC is up (port 19333)"
|
||||
break
|
||||
fi
|
||||
echo " Waiting for master gRPC... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Give server time to initialize all components including gRPC services
|
||||
echo "Waiting for SeaweedFS components to initialize..."
|
||||
sleep 15
|
||||
|
||||
# Additional wait specifically for gRPC services to be ready for streaming
|
||||
echo "Allowing extra time for master gRPC streaming services to initialize..."
|
||||
sleep 10
|
||||
|
||||
# Start MQ broker with maximum verbosity for debugging
|
||||
echo "Starting MQ broker..."
|
||||
nohup weed -v 3 mq.broker \
|
||||
-master="127.0.0.1:9333" \
|
||||
-ip="127.0.0.1" \
|
||||
-port=17777 \
|
||||
-logFlushInterval=0 \
|
||||
> /tmp/weed-mq-broker.log 2>&1 &
|
||||
|
||||
# Wait for broker to be ready with better error reporting
|
||||
sleep 15
|
||||
broker_ready=false
|
||||
for i in $(seq 1 20); do
|
||||
if nc -z 127.0.0.1 17777; then
|
||||
echo "SeaweedFS MQ broker is up"
|
||||
broker_ready=true
|
||||
break
|
||||
fi
|
||||
echo "Waiting for MQ broker... ($i/20)"; sleep 1
|
||||
done
|
||||
|
||||
# Give broker additional time to register with master
|
||||
if [ "$broker_ready" = true ]; then
|
||||
echo "Allowing broker to register with master..."
|
||||
sleep 30
|
||||
|
||||
# Check if broker is properly registered by querying cluster nodes
|
||||
echo "Cluster status after broker registration:"
|
||||
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
|
||||
|
||||
echo "Checking cluster topology (includes registered components):"
|
||||
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
|
||||
|
||||
echo "Verifying broker discovery via master client debug:"
|
||||
echo "If broker registration is successful, it should appear in dir status"
|
||||
|
||||
echo "Testing gRPC connectivity with weed binary:"
|
||||
echo "This simulates what the gateway does during broker discovery..."
|
||||
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
|
||||
echo "Shell test results:"
|
||||
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
|
||||
fi
|
||||
|
||||
# Check if broker failed to start and show logs
|
||||
if [ "$broker_ready" = false ]; then
|
||||
echo "ERROR: MQ broker failed to start. Broker logs:"
|
||||
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
|
||||
echo "Server logs:"
|
||||
tail -20 /tmp/weed-server.log || echo "No server logs found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run SMQ Integration Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
# Run the dedicated SMQ integration tests
|
||||
go test -v -run "^TestSMQIntegration" -timeout 180s ./integration/...
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
kafka-protocol-tests:
|
||||
name: Kafka Protocol Tests (Isolated)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [protocol-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
options: --cpus 1.0 --memory 1g --tmpfs /tmp:exec --hostname kafka-protocol-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_PROTOCOL_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Protocol Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps
|
||||
# Ensure proper permissions for test execution
|
||||
chmod -R 755 /tmp || true
|
||||
export TMPDIR=/tmp
|
||||
export GOCACHE=/tmp/go-cache
|
||||
mkdir -p $GOCACHE
|
||||
chmod 755 $GOCACHE
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Protocol Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
export TMPDIR=/tmp
|
||||
export GOCACHE=/tmp/go-cache
|
||||
# Run protocol tests from the weed/mq/kafka directory since they test the protocol implementation
|
||||
cd ../../weed/mq/kafka
|
||||
go test -v -run "^Test.*" -timeout 10s ./...
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
TMPDIR: /tmp
|
||||
GOCACHE: /tmp/go-cache
|
||||
@@ -1,92 +0,0 @@
|
||||
name: "Metadata Subscribe Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/filer/**'
|
||||
- 'weed/pb/filer_pb/**'
|
||||
- 'weed/util/log_buffer/**'
|
||||
- 'weed/server/filer_grpc_server_sub_meta.go'
|
||||
- 'weed/command/filer_backup.go'
|
||||
- 'test/metadata_subscribe/**'
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/filer/**'
|
||||
- 'weed/pb/filer_pb/**'
|
||||
- 'weed/util/log_buffer/**'
|
||||
- 'weed/server/filer_grpc_server_sub_meta.go'
|
||||
- 'weed/command/filer_backup.go'
|
||||
- 'test/metadata_subscribe/**'
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/metadata-subscribe-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '10m'
|
||||
|
||||
jobs:
|
||||
metadata-subscribe-integration:
|
||||
name: Metadata Subscribe Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run Metadata Subscribe Integration Tests
|
||||
run: |
|
||||
cd test/metadata_subscribe
|
||||
|
||||
echo "Running Metadata Subscribe integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Run tests with verbose output
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "Metadata Subscribe integration tests completed"
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: metadata-subscribe-test-logs
|
||||
path: |
|
||||
/tmp/seaweedfs_*
|
||||
retention-days: 7
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Metadata Subscribe Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Basic Subscription**: Subscribe to metadata changes and receive events" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Single-Filer No Stall**: Regression test for issue #4977" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Resume from Disk**: Verify subscription can resume from persisted logs" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Related Issues" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- [#4977](https://github.com/seaweedfs/seaweedfs/issues/4977): filer.backup synchronisation stall" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
name: "PostgreSQL Gateway Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/postgres-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
postgres-basic-tests:
|
||||
name: PostgreSQL Basic Tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
defaults:
|
||||
run:
|
||||
working-directory: test/postgres
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-postgres-
|
||||
|
||||
- name: Start PostgreSQL Gateway Services
|
||||
run: |
|
||||
make dev-start
|
||||
sleep 10
|
||||
|
||||
- name: Run Basic Connectivity Test
|
||||
run: |
|
||||
make test-basic
|
||||
|
||||
- name: Run PostgreSQL Client Tests
|
||||
run: |
|
||||
make test-client
|
||||
|
||||
- name: Save logs
|
||||
if: always()
|
||||
run: |
|
||||
docker compose logs > postgres-output.log || true
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: postgres-logs
|
||||
path: test/postgres/postgres-output.log
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
make clean || true
|
||||
@@ -1,86 +0,0 @@
|
||||
name: "S3 Filer Group Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-filer-group-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
s3-filer-group-tests:
|
||||
name: S3 Filer Group Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Filer Group Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/filer_group
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Filer Group Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
# The test-with-server target handles server startup/shutdown automatically
|
||||
# Server is started with -filer.group=testgroup
|
||||
make test-with-server || {
|
||||
echo "❌ Filer group tests failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/filer_group
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-filer-group-test-logs
|
||||
path: test/s3/filer_group/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
|
||||
@@ -25,10 +25,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-versioning-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -89,10 +89,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-versioning-compatibility-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -137,10 +137,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-cors-compatibility-logs
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -188,10 +188,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -239,7 +239,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-retention-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -255,10 +255,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -306,7 +306,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-cors-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -319,10 +319,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -355,7 +355,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-retention-worm-logs
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -365,13 +365,15 @@ jobs:
|
||||
name: S3 Versioning Stress Test
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
# Only run stress tests on master branch pushes to avoid overloading PR testing
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -389,9 +391,6 @@ jobs:
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Start server for stress tests
|
||||
make start-server
|
||||
|
||||
# Run stress tests (concurrent operations)
|
||||
make test-versioning-stress || {
|
||||
echo "❌ Stress test failed, checking logs..."
|
||||
@@ -399,142 +398,17 @@ jobs:
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make stop-server
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Run pagination stress tests (>1000 versions)
|
||||
echo "=== Running pagination stress tests ==="
|
||||
make test-versioning-pagination-stress || {
|
||||
echo "❌ Pagination stress test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make stop-server
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
|
||||
make stop-server
|
||||
make clean
|
||||
|
||||
- name: Upload stress test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-versioning-stress-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 7
|
||||
|
||||
s3-tagging-tests:
|
||||
# CI job for S3 object tagging tests
|
||||
name: S3 Tagging Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Tagging Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/tagging
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Set environment variables for the test
|
||||
export S3_ENDPOINT="http://localhost:8006"
|
||||
export S3_ACCESS_KEY="0555b35654ad1656d804"
|
||||
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
|
||||
|
||||
# Run the specific test that is equivalent to AWS S3 tagging behavior
|
||||
make test-with-server || {
|
||||
echo "❌ Test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-tagging-test-logs
|
||||
path: test/s3/tagging/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-remote-cache-tests:
|
||||
name: S3 Remote Cache Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Remote Cache Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/remote_cache
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run the remote cache integration tests
|
||||
# Tests singleflight deduplication for caching remote objects
|
||||
make test-with-server || {
|
||||
echo "❌ Test failed, checking logs..."
|
||||
if [ -f primary-weed.log ]; then
|
||||
echo "=== Primary server logs ==="
|
||||
tail -100 primary-weed.log
|
||||
fi
|
||||
if [ -f remote-weed.log ]; then
|
||||
echo "=== Remote server logs ==="
|
||||
tail -100 remote-weed.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-remote-cache-test-logs
|
||||
path: |
|
||||
test/s3/remote_cache/primary-weed.log
|
||||
test/s3/remote_cache/remote-weed.log
|
||||
retention-days: 3
|
||||
|
||||
# Removed SSE-C integration tests and compatibility job
|
||||
@@ -35,10 +35,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: iam-unit-test-results
|
||||
path: |
|
||||
@@ -84,10 +84,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -162,7 +162,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-iam-integration-logs-${{ matrix.test-type }}
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -176,10 +176,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -222,7 +222,7 @@ jobs:
|
||||
|
||||
- name: Upload distributed test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-iam-distributed-logs
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -236,10 +236,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -274,7 +274,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-iam-performance-results
|
||||
path: |
|
||||
|
||||
@@ -35,10 +35,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-keycloak-test-logs
|
||||
path: |
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
name: "S3 PyArrow Parquet Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/s3/parquet/**'
|
||||
- '.github/workflows/s3-parquet-tests.yml'
|
||||
pull_request:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/s3/parquet/**'
|
||||
- '.github/workflows/s3-parquet-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
S3_ACCESS_KEY: some_access_key1
|
||||
S3_SECRET_KEY: some_secret_key1
|
||||
S3_ENDPOINT_URL: http://localhost:8333
|
||||
BUCKET_NAME: test-parquet-bucket
|
||||
|
||||
jobs:
|
||||
parquet-integration-tests:
|
||||
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ['3.9', '3.11', '3.12']
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'test/s3/parquet/requirements.txt'
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y lsof netcat-openbsd
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -v
|
||||
sudo cp weed /usr/local/bin/
|
||||
weed version
|
||||
|
||||
- name: Run PyArrow Parquet integration tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make test-with-server
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
VOLUME_MAX_SIZE_MB: 50
|
||||
|
||||
- name: Run implicit directory fix tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make test-implicit-dir-with-server
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
|
||||
- name: Run PyArrow native S3 filesystem tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make test-native-s3-with-server
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
|
||||
- name: Run cross-filesystem compatibility tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
TEST_QUICK=1 make test-cross-fs-with-server
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
|
||||
- name: Run SSE-S3 encryption compatibility tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make test-sse-s3-compat
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: test-logs-python-${{ matrix.python-version }}
|
||||
path: |
|
||||
/tmp/seaweedfs-parquet-*.log
|
||||
test/s3/parquet/*.log
|
||||
retention-days: 7
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make stop-seaweedfs-safe || true
|
||||
make clean || true
|
||||
|
||||
unit-tests:
|
||||
name: Go Unit Tests (Implicit Directory)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
|
||||
- name: Run Go unit tests
|
||||
run: |
|
||||
cd weed/s3api
|
||||
go test -v -run TestImplicitDirectory
|
||||
|
||||
- name: Run all S3 API tests
|
||||
run: |
|
||||
cd weed/s3api
|
||||
go test -v -timeout 5m
|
||||
|
||||
@@ -4,7 +4,6 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'weed/s3api/s3api_object_handlers_copy*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
@@ -15,7 +14,6 @@ on:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'weed/s3api/s3api_object_handlers_copy*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
@@ -44,10 +42,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -70,11 +68,11 @@ jobs:
|
||||
# Run tests with automatic server management
|
||||
# The test-with-server target handles server startup/shutdown automatically
|
||||
if [ "${{ matrix.test-type }}" = "quick" ]; then
|
||||
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
# Quick tests - basic SSE-C and SSE-KMS functionality
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration"
|
||||
else
|
||||
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration"
|
||||
fi
|
||||
|
||||
- name: Show server logs on failure
|
||||
@@ -95,7 +93,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-sse-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -108,10 +106,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -129,8 +127,8 @@ jobs:
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality plus Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|.*RangeRequestsServerBehavior" || {
|
||||
# Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic" || {
|
||||
echo "❌ SSE compatibility test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
@@ -143,7 +141,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-sse-compatibility-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -156,10 +154,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -192,7 +190,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-sse-metadata-persistence-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -205,10 +203,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -241,7 +239,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-sse-copy-operations-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -254,10 +252,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -290,7 +288,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-sse-multipart-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -305,10 +303,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
@@ -340,58 +338,8 @@ jobs:
|
||||
|
||||
- name: Upload performance test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-sse-performance-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
retention-days: 7
|
||||
|
||||
s3-volume-encryption:
|
||||
name: S3 Volume Encryption Test
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Volume Encryption Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run volume encryption tests with -s3.encryptVolumeData flag
|
||||
echo "🚀 Running S3 volume encryption integration tests..."
|
||||
make test-volume-encryption || {
|
||||
echo "❌ Volume encryption tests failed, checking logs..."
|
||||
if [ -f /tmp/seaweedfs-sse-mini.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 /tmp/seaweedfs-sse-mini.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-volume-encryption-logs
|
||||
path: /tmp/seaweedfs-sse-*.log
|
||||
retention-days: 3
|
||||
|
||||
|
||||
+393
-448
File diff suppressed because it is too large
Load Diff
@@ -1,94 +0,0 @@
|
||||
name: "SFTP Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/sftpd/**'
|
||||
- 'weed/command/sftp.go'
|
||||
- 'test/sftp/**'
|
||||
- '.github/workflows/sftp-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/sftpd/**'
|
||||
- 'weed/command/sftp.go'
|
||||
- 'test/sftp/**'
|
||||
- '.github/workflows/sftp-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/sftp-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
sftp-integration:
|
||||
name: SFTP Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y openssh-client
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run SFTP Integration Tests
|
||||
run: |
|
||||
cd test/sftp
|
||||
|
||||
echo "🧪 Running SFTP integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run all SFTP tests
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "✅ SFTP integration tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## 🔐 SFTP Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **HomeDir Path Translation**: User home directory mapping (fixes #7470)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **File Operations**: Upload, download, delete" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Directory Operations**: Create, list, remove" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Large File Handling**: 1MB+ file support" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Path Edge Cases**: Unicode, trailing slashes, .. paths" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Admin Access**: Root user verification" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Configuration" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| User | HomeDir | Permissions |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|------|---------|-------------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| admin | / | Full access |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| testuser | /sftp/testuser | Home directory only |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
|
||||
|
||||
@@ -1,263 +0,0 @@
|
||||
name: Spark Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'test/java/spark/**'
|
||||
- 'other/java/hdfs3/**'
|
||||
- 'other/java/client/**'
|
||||
- '.github/workflows/spark-integration-tests.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'test/java/spark/**'
|
||||
- 'other/java/hdfs3/**'
|
||||
- 'other/java/client/**'
|
||||
- '.github/workflows/spark-integration-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
spark-integration-tests:
|
||||
name: Spark Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
# ========================================
|
||||
# SETUP & BUILD
|
||||
# ========================================
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: '11'
|
||||
distribution: 'temurin'
|
||||
cache: maven
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
echo "Building SeaweedFS binary (statically linked for Alpine)..."
|
||||
cd weed
|
||||
CGO_ENABLED=0 go build -o ../docker/weed
|
||||
cd ../docker
|
||||
ls -la weed filer.toml entrypoint.sh
|
||||
file weed
|
||||
echo "OK SeaweedFS binary built"
|
||||
|
||||
- name: Build SeaweedFS Java dependencies
|
||||
run: |
|
||||
echo "Building Java client..."
|
||||
cd other/java/client
|
||||
mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
|
||||
echo "OK Java client built"
|
||||
cd ../../..
|
||||
|
||||
echo "Building HDFS3 client..."
|
||||
cd other/java/hdfs3
|
||||
mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
|
||||
echo "OK HDFS3 client built"
|
||||
echo ""
|
||||
echo "All Java dependencies installed to ~/.m2/repository"
|
||||
|
||||
# ========================================
|
||||
# SPARK INTEGRATION TESTS (DOCKER)
|
||||
# ========================================
|
||||
- name: Start SeaweedFS services for tests
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Starting SeaweedFS Services for Tests ==="
|
||||
docker compose down -v || true
|
||||
docker compose build --no-cache
|
||||
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
|
||||
|
||||
echo "Waiting for services..."
|
||||
for i in {1..30}; do
|
||||
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
|
||||
echo "OK SeaweedFS filer is ready!"
|
||||
break
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "FAILED Services failed to start"
|
||||
docker compose ps -a
|
||||
docker compose logs
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
curl -f http://localhost:9333/cluster/status || exit 1
|
||||
echo "OK All services healthy"
|
||||
|
||||
- name: Prepare Maven repository for Docker
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "Copying Maven artifacts for Docker container..."
|
||||
mkdir -p .m2/repository/com
|
||||
cp -r ~/.m2/repository/com/seaweedfs .m2/repository/com/
|
||||
echo "OK Maven artifacts copied"
|
||||
|
||||
- name: Run Spark integration tests
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Running Spark Integration Tests ==="
|
||||
docker compose up --abort-on-container-exit spark-tests
|
||||
|
||||
echo ""
|
||||
echo "=== Test Logs ==="
|
||||
docker compose logs spark-tests | tail -100
|
||||
|
||||
- name: Stop test services
|
||||
if: always()
|
||||
working-directory: test/java/spark
|
||||
run: docker compose down -v
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: spark-test-results
|
||||
path: test/java/spark/target/surefire-reports/
|
||||
retention-days: 30
|
||||
|
||||
- name: Publish test report
|
||||
if: always()
|
||||
uses: dorny/test-reporter@v2
|
||||
with:
|
||||
name: Spark Test Results
|
||||
path: test/java/spark/target/surefire-reports/*.xml
|
||||
reporter: java-junit
|
||||
fail-on-error: true
|
||||
|
||||
# ========================================
|
||||
# SPARK EXAMPLE (HOST-BASED) - DISABLED
|
||||
# Note: Host-based example doesn't work with Docker networking
|
||||
# because master returns Docker hostnames (seaweedfs-volume)
|
||||
# which are not resolvable from the host.
|
||||
# The Docker-based tests above are sufficient.
|
||||
# ========================================
|
||||
- name: Cache Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
id: cache-spark
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: spark-3.5.0-bin-hadoop3
|
||||
key: spark-3.5.0-hadoop3
|
||||
|
||||
- name: Download Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && steps.cache-spark.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
echo "Downloading Apache Spark 3.5.0..."
|
||||
wget -q https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz
|
||||
tar xzf spark-3.5.0-bin-hadoop3.tgz
|
||||
echo "OK Spark downloaded"
|
||||
|
||||
- name: Start SeaweedFS services for example
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Starting SeaweedFS Services for Example ==="
|
||||
docker compose down -v || true
|
||||
docker compose build --no-cache
|
||||
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
|
||||
|
||||
echo "Waiting for services..."
|
||||
for i in {1..30}; do
|
||||
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
|
||||
echo "OK SeaweedFS filer is ready!"
|
||||
break
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "FAILED Services failed to start"
|
||||
docker compose ps -a
|
||||
docker compose logs
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
curl -f http://localhost:9333/cluster/status || exit 1
|
||||
echo "OK All services healthy"
|
||||
|
||||
- name: Clean target directory
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
# Force remove target directory to avoid permission issues
|
||||
sudo rm -rf target || rm -rf target || true
|
||||
echo "OK Target directory cleaned"
|
||||
|
||||
- name: Build project for example
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: mvn package -DskipTests
|
||||
|
||||
- name: Run Spark example application
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Running Spark Example Application ==="
|
||||
export SPARK_HOME=$(pwd)/../../../spark-3.5.0-bin-hadoop3
|
||||
$SPARK_HOME/bin/spark-submit \
|
||||
--class seaweed.spark.SparkSeaweedFSExample \
|
||||
--master local[2] \
|
||||
--conf spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem \
|
||||
--conf spark.hadoop.fs.seaweed.filer.host=localhost \
|
||||
--conf spark.hadoop.fs.seaweed.filer.port=8888 \
|
||||
--conf spark.hadoop.fs.seaweed.filer.port.grpc=18888 \
|
||||
--conf spark.hadoop.fs.seaweed.replication="" \
|
||||
target/seaweedfs-spark-integration-tests-1.0-SNAPSHOT.jar \
|
||||
seaweedfs://localhost:8888/ci-spark-output
|
||||
echo "OK Example completed"
|
||||
|
||||
- name: Verify example output
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
run: |
|
||||
echo "Verifying output..."
|
||||
curl -s http://localhost:8888/ci-spark-output/ || echo "Output listing unavailable"
|
||||
|
||||
- name: Stop example services
|
||||
if: false && always() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: docker compose down -v
|
||||
|
||||
# ========================================
|
||||
# DIAGNOSTICS
|
||||
# ========================================
|
||||
- name: Display diagnostics on failure
|
||||
if: failure()
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Container Status ==="
|
||||
docker compose ps -a
|
||||
echo ""
|
||||
echo "=== Master Logs ==="
|
||||
docker compose logs seaweedfs-master
|
||||
echo ""
|
||||
echo "=== Volume Logs ==="
|
||||
docker compose logs seaweedfs-volume
|
||||
echo ""
|
||||
echo "=== Filer Logs ==="
|
||||
docker compose logs seaweedfs-filer
|
||||
echo ""
|
||||
echo "=== Volume List ==="
|
||||
docker compose exec -T seaweedfs-master weed shell <<EOF || echo "Failed"
|
||||
volume.list
|
||||
exit
|
||||
EOF
|
||||
echo ""
|
||||
echo "=== Cluster Status ==="
|
||||
curl -s http://localhost:9333/dir/status | jq '.' || curl -s http://localhost:9333/dir/status
|
||||
|
||||
@@ -18,11 +18,11 @@ defaults:
|
||||
jobs:
|
||||
awscli-tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ^1.24
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
run: |
|
||||
set -e
|
||||
mkdir -p /tmp/data
|
||||
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
|
||||
./weed server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json &
|
||||
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Setup Caddy
|
||||
@@ -77,35 +77,3 @@ jobs:
|
||||
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded
|
||||
diff -q generated downloaded
|
||||
rm -f generated downloaded
|
||||
|
||||
- name: Test GetObject with If-Match
|
||||
run: |
|
||||
set -e
|
||||
dd if=/dev/urandom of=generated bs=1M count=32
|
||||
ETAG=$(aws --no-verify-ssl s3api put-object --bucket bucket --key test-get-obj --body generated | jq -r .ETag)
|
||||
# jq -r already removes quotes, so use ETAG directly (handles both simple and multipart ETags)
|
||||
aws --no-verify-ssl s3api get-object --bucket bucket --key test-get-obj --if-match "$ETAG" downloaded
|
||||
diff -q generated downloaded
|
||||
rm -f generated downloaded
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "========================================="
|
||||
echo "SeaweedFS Server Logs"
|
||||
echo "========================================="
|
||||
# Note: weed.log is relative to working-directory (weed/)
|
||||
if [ -f weed.log ]; then
|
||||
cat weed.log
|
||||
else
|
||||
echo "No weed.log file found"
|
||||
fi
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: seaweedfs-logs
|
||||
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
|
||||
path: weed/weed.log
|
||||
retention-days: 3
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
name: "TUS Protocol Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
tus-integration-tests:
|
||||
name: TUS Protocol Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run TUS Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/tus
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting TUS Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
make test-with-server || {
|
||||
echo "TUS integration tests failed, checking logs..."
|
||||
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
|
||||
echo "=== Filer logs ==="
|
||||
tail -100 /tmp/seaweedfs-tus-filer.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs-tus-master.log ]; then
|
||||
echo "=== Master logs ==="
|
||||
tail -50 /tmp/seaweedfs-tus-master.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
|
||||
echo "=== Volume logs ==="
|
||||
tail -50 /tmp/seaweedfs-tus-volume.log
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/tus
|
||||
run: |
|
||||
echo "=== Filer Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
|
||||
echo "Last 100 lines of filer logs:"
|
||||
tail -100 /tmp/seaweedfs-tus-filer.log
|
||||
else
|
||||
echo "No filer log file found"
|
||||
fi
|
||||
|
||||
echo "=== Master Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-master.log ]; then
|
||||
tail -50 /tmp/seaweedfs-tus-master.log
|
||||
else
|
||||
echo "No master log file found"
|
||||
fi
|
||||
|
||||
echo "=== Volume Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
|
||||
tail -50 /tmp/seaweedfs-tus-volume.log
|
||||
else
|
||||
echo "No volume log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(18888|19333|18080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tus-test-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-tus-*.log
|
||||
retention-days: 3
|
||||
+10
-21
@@ -100,21 +100,24 @@ test/mq/bin/producer
|
||||
test/producer
|
||||
bin/weed
|
||||
weed_binary
|
||||
/test/s3/copying/filerldb2
|
||||
/filerldb2
|
||||
/test/s3/retention/test-volume-data
|
||||
test/s3/copying/filerldb2
|
||||
filerldb2
|
||||
test/s3/retention/test-volume-data
|
||||
test/s3/cors/weed-test.log
|
||||
test/s3/cors/weed-server.pid
|
||||
/test/s3/cors/test-volume-data
|
||||
test/s3/cors/test-volume-data
|
||||
test/s3/cors/cors.test
|
||||
/test/s3/retention/filerldb2
|
||||
test/s3/retention/filerldb2
|
||||
test/s3/retention/weed-server.pid
|
||||
test/s3/retention/weed-test.log
|
||||
/test/s3/versioning/test-volume-data
|
||||
test/s3/versioning/test-volume-data
|
||||
test/s3/versioning/weed-test.log
|
||||
/docker/admin_integration/data
|
||||
docker/admin_integration/data
|
||||
docker/agent_pub_record
|
||||
docker/admin_integration/weed-local
|
||||
docker/admin_integration/ec_test_files.json
|
||||
docker/admin_integration/data1
|
||||
seaweedfs-rdma-sidecar/bin
|
||||
/seaweedfs-rdma-sidecar/bin
|
||||
/test/s3/encryption/filerldb2
|
||||
/test/s3/sse/filerldb2
|
||||
@@ -123,17 +126,3 @@ ADVANCED_IAM_DEVELOPMENT_PLAN.md
|
||||
/test/s3/iam/test-volume-data
|
||||
*.log
|
||||
weed-iam
|
||||
test/kafka/kafka-client-loadtest/weed-linux-arm64
|
||||
/test/tus/filerldb2
|
||||
coverage.out
|
||||
/test/s3/remote_cache/test-primary-data
|
||||
/test/s3/remote_cache/test-remote-data
|
||||
test/s3/remote_cache/remote-server.pid
|
||||
test/s3/remote_cache/primary-server.pid
|
||||
/test/erasure_coding/filerldb2
|
||||
/test/s3/cors/test-mini-data
|
||||
/test/s3/filer_group/test-volume-data
|
||||
|
||||
# ID and PID files
|
||||
*.id
|
||||
*.pid
|
||||
|
||||
@@ -1,242 +0,0 @@
|
||||
# Bucket Policy Engine Integration - Complete
|
||||
|
||||
## Summary
|
||||
|
||||
Successfully integrated the `policy_engine` package to evaluate bucket policies for **all requests** (both anonymous and authenticated). This provides comprehensive AWS S3-compatible bucket policy support.
|
||||
|
||||
## What Changed
|
||||
|
||||
### 1. **New File: `s3api_bucket_policy_engine.go`**
|
||||
Created a wrapper around `policy_engine.PolicyEngine` to:
|
||||
- Load bucket policies from filer entries
|
||||
- Sync policies from the bucket config cache
|
||||
- Evaluate policies for any request (bucket, object, action, principal)
|
||||
- Return structured results (allowed, evaluated, error)
|
||||
|
||||
### 2. **Modified: `s3api_server.go`**
|
||||
- Added `policyEngine *BucketPolicyEngine` field to `S3ApiServer` struct
|
||||
- Initialized the policy engine in `NewS3ApiServerWithStore()`
|
||||
- Linked `IdentityAccessManagement` back to `S3ApiServer` for policy evaluation
|
||||
|
||||
### 3. **Modified: `auth_credentials.go`**
|
||||
- Added `s3ApiServer *S3ApiServer` field to `IdentityAccessManagement` struct
|
||||
- Added `buildPrincipalARN()` helper to convert identities to AWS ARN format
|
||||
- **Integrated bucket policy evaluation into the authentication flow:**
|
||||
- Policies are now checked **before** IAM/identity-based permissions
|
||||
- Explicit `Deny` in bucket policy blocks access immediately
|
||||
- Explicit `Allow` in bucket policy grants access and **bypasses IAM checks** (enables cross-account access)
|
||||
- If no policy exists, falls through to normal IAM checks
|
||||
- Policy evaluation errors result in access denial (fail-close security)
|
||||
|
||||
### 4. **Modified: `s3api_bucket_config.go`**
|
||||
- Added policy engine sync when bucket configs are loaded
|
||||
- Ensures policies are loaded into the engine for evaluation
|
||||
|
||||
### 5. **Modified: `auth_credentials_subscribe.go`**
|
||||
- Added policy engine sync when bucket metadata changes
|
||||
- Keeps the policy engine up-to-date via event-driven updates
|
||||
|
||||
## How It Works
|
||||
|
||||
### Anonymous Requests
|
||||
```
|
||||
1. Request comes in (no credentials)
|
||||
2. Check ACL-based public access → if public, allow
|
||||
3. Check bucket policy for anonymous ("*") access → if allowed, allow
|
||||
4. Otherwise, deny
|
||||
```
|
||||
|
||||
### Authenticated Requests (NEW!)
|
||||
```
|
||||
1. Request comes in (with credentials)
|
||||
2. Authenticate user → get Identity
|
||||
3. Build principal ARN (e.g., "arn:aws:iam::123456:user/bob")
|
||||
4. Check bucket policy:
|
||||
- If DENY → reject immediately
|
||||
- If ALLOW → grant access immediately (bypasses IAM checks)
|
||||
- If no policy or no matching statements → continue to step 5
|
||||
5. Check IAM/identity-based permissions (only if not already allowed by bucket policy)
|
||||
6. Allow or deny based on identity permissions
|
||||
```
|
||||
|
||||
## Policy Evaluation Flow
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Request (GET /bucket/file) │
|
||||
└───────────────────────────┬─────────────────────────────┘
|
||||
│
|
||||
┌───────────▼──────────┐
|
||||
│ Authenticate User │
|
||||
│ (or Anonymous) │
|
||||
└───────────┬──────────┘
|
||||
│
|
||||
┌───────────▼──────────────────────────────┐
|
||||
│ Build Principal ARN │
|
||||
│ - Anonymous: "*" │
|
||||
│ - User: "arn:aws:iam::123456:user/bob" │
|
||||
└───────────┬──────────────────────────────┘
|
||||
│
|
||||
┌───────────▼──────────────────────────────┐
|
||||
│ Evaluate Bucket Policy (PolicyEngine) │
|
||||
│ - Action: "s3:GetObject" │
|
||||
│ - Resource: "arn:aws:s3:::bucket/file" │
|
||||
│ - Principal: (from above) │
|
||||
└───────────┬──────────────────────────────┘
|
||||
│
|
||||
┌─────────────┼─────────────┐
|
||||
│ │ │
|
||||
DENY │ ALLOW │ NO POLICY
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
Reject Request Grant Access Continue
|
||||
│
|
||||
┌───────────────────┘
|
||||
│
|
||||
┌────────────▼─────────────┐
|
||||
│ IAM/Identity Check │
|
||||
│ (identity.canDo) │
|
||||
└────────────┬─────────────┘
|
||||
│
|
||||
┌─────────┴─────────┐
|
||||
│ │
|
||||
ALLOW │ DENY │
|
||||
▼ ▼
|
||||
Grant Access Reject Request
|
||||
```
|
||||
|
||||
## Example Policies That Now Work
|
||||
|
||||
### 1. **Public Read Access** (Anonymous)
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::mybucket/*"
|
||||
}]
|
||||
}
|
||||
```
|
||||
- Anonymous users can read all objects
|
||||
- Authenticated users are also evaluated against this policy. If they don't match an explicit `Allow` for this action, they will fall back to their own IAM permissions
|
||||
|
||||
### 2. **Grant Access to Specific User** (Authenticated)
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": {"AWS": "arn:aws:iam::123456789012:user/bob"},
|
||||
"Action": ["s3:GetObject", "s3:PutObject"],
|
||||
"Resource": "arn:aws:s3:::mybucket/shared/*"
|
||||
}]
|
||||
}
|
||||
```
|
||||
- User "bob" can read/write objects in `/shared/` prefix
|
||||
- Other users cannot (unless granted by their IAM policies)
|
||||
|
||||
### 3. **Deny Access to Specific Path** (Both)
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny",
|
||||
"Principal": "*",
|
||||
"Action": "s3:*",
|
||||
"Resource": "arn:aws:s3:::mybucket/confidential/*"
|
||||
}]
|
||||
}
|
||||
```
|
||||
- **No one** can access `/confidential/` objects
|
||||
- Denies override all other allows (AWS policy evaluation rules)
|
||||
|
||||
## Performance Characteristics
|
||||
|
||||
### Policy Loading
|
||||
- **Cold start**: Policy loaded from filer → parsed → compiled → cached
|
||||
- **Warm path**: Policy retrieved from `BucketConfigCache` (already parsed)
|
||||
- **Updates**: Event-driven sync via metadata subscription (real-time)
|
||||
|
||||
### Policy Evaluation
|
||||
- **Compiled policies**: Pre-compiled regex patterns and matchers
|
||||
- **Pattern cache**: Regex patterns cached with LRU eviction (max 1000)
|
||||
- **Fast path**: Common patterns (`*`, exact matches) optimized
|
||||
- **Case sensitivity**: Actions case-insensitive, resources case-sensitive (AWS-compatible)
|
||||
|
||||
### Overhead
|
||||
- **Anonymous requests**: Minimal (policy already checked, now using compiled engine)
|
||||
- **Authenticated requests**: ~1-2ms added for policy evaluation (compiled patterns)
|
||||
- **No policy**: Near-zero overhead (quick indeterminate check)
|
||||
|
||||
## Testing
|
||||
|
||||
All tests pass:
|
||||
```bash
|
||||
✅ TestBucketPolicyValidationBasics
|
||||
✅ TestPrincipalMatchesAnonymous
|
||||
✅ TestActionToS3Action
|
||||
✅ TestResourceMatching
|
||||
✅ TestMatchesPatternRegexEscaping (security tests)
|
||||
✅ TestActionMatchingCaseInsensitive
|
||||
✅ TestResourceMatchingCaseSensitive
|
||||
✅ All policy_engine package tests (30+ tests)
|
||||
```
|
||||
|
||||
## Security Improvements
|
||||
|
||||
1. **Regex Metacharacter Escaping**: Patterns like `*.json` properly match only files ending in `.json` (not `filexjson`)
|
||||
2. **Case-Insensitive Actions**: S3 actions matched case-insensitively per AWS spec
|
||||
3. **Case-Sensitive Resources**: Resource paths matched case-sensitively for security
|
||||
4. **Pattern Cache Size Limit**: Prevents DoS attacks via unbounded cache growth
|
||||
5. **Principal Validation**: Supports `[]string` for manually constructed policies
|
||||
|
||||
## AWS Compatibility
|
||||
|
||||
The implementation follows AWS S3 bucket policy evaluation rules:
|
||||
1. **Explicit Deny** always wins (checked first)
|
||||
2. **Explicit Allow** grants access (checked second)
|
||||
3. **Default Deny** if no matching statements (implicit)
|
||||
4. Bucket policies work alongside IAM policies (both are evaluated)
|
||||
|
||||
## Files Changed
|
||||
|
||||
```
|
||||
Modified:
|
||||
weed/s3api/auth_credentials.go (+47 lines)
|
||||
weed/s3api/auth_credentials_subscribe.go (+8 lines)
|
||||
weed/s3api/s3api_bucket_config.go (+8 lines)
|
||||
weed/s3api/s3api_server.go (+5 lines)
|
||||
|
||||
New:
|
||||
weed/s3api/s3api_bucket_policy_engine.go (115 lines)
|
||||
```
|
||||
|
||||
## Migration Notes
|
||||
|
||||
- **Backward Compatible**: Existing setups without bucket policies work unchanged
|
||||
- **No Breaking Changes**: All existing ACL and IAM-based authorization still works
|
||||
- **Additive Feature**: Bucket policies are an additional layer of authorization
|
||||
- **Performance**: Minimal impact on existing workloads
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
Potential improvements (not implemented yet):
|
||||
- [ ] Condition support (IP address, time-based, etc.) - already in policy_engine
|
||||
- [ ] Cross-account policies (different AWS accounts)
|
||||
- [ ] Policy validation API endpoint
|
||||
- [ ] Policy simulation/testing tool
|
||||
- [ ] Metrics for policy evaluations (allow/deny counts)
|
||||
|
||||
## Conclusion
|
||||
|
||||
Bucket policies now work for **all requests** in SeaweedFS S3 API:
|
||||
- ✅ Anonymous requests (public access)
|
||||
- ✅ Authenticated requests (user-specific policies)
|
||||
- ✅ High performance (compiled policies, caching)
|
||||
- ✅ AWS-compatible (follows AWS evaluation rules)
|
||||
- ✅ Secure (proper escaping, case sensitivity)
|
||||
|
||||
The integration is complete, tested, and ready for use!
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# Build Notes for SeaweedFS Development
|
||||
|
||||
## Protobuf Generation
|
||||
|
||||
To regenerate protobuf Go files after modifying .proto files:
|
||||
|
||||
```bash
|
||||
cd weed/pb
|
||||
make
|
||||
```
|
||||
|
||||
This will regenerate all the protobuf Go files with the latest changes from the .proto definitions.
|
||||
|
||||
## Other Important Build Commands
|
||||
|
||||
- Main build: `make`
|
||||
- Clean build: `make clean && make`
|
||||
- Tests: `make test`
|
||||
|
||||
---
|
||||
*Generated: This file contains important build commands for development*
|
||||
@@ -18,12 +18,12 @@ full_install: admin-generate
|
||||
cd weed; go install -tags "elastic gocdk sqlite ydb tarantool tikv rclone"
|
||||
|
||||
server: install
|
||||
weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
|
||||
weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
|
||||
|
||||
benchmark: install warp_install
|
||||
pkill weed || true
|
||||
pkill warp || true
|
||||
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
|
||||
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
|
||||
warp client &
|
||||
while ! nc -z localhost 8000 ; do sleep 1 ; done
|
||||
warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
|
||||
|
||||
@@ -54,9 +54,9 @@ Table of Contents
|
||||
=================
|
||||
|
||||
* [Quick Start](#quick-start)
|
||||
* [Quick Start with weed mini](#quick-start-with-weed-mini)
|
||||
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
|
||||
* [Quick Start with Single Binary](#quick-start-with-single-binary)
|
||||
* [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws)
|
||||
* [Introduction](#introduction)
|
||||
* [Features](#features)
|
||||
* [Additional Features](#additional-features)
|
||||
@@ -78,32 +78,6 @@ Table of Contents
|
||||
|
||||
# Quick Start #
|
||||
|
||||
|
||||
## Quick Start with weed mini ##
|
||||
The easiest way to get started with SeaweedFS for development and testing:
|
||||
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
# remove quarantine on macOS
|
||||
# xattr -d com.apple.quarantine ./weed
|
||||
export AWS_ACCESS_KEY_ID=admin
|
||||
export AWS_SECRET_ACCESS_KEY=key
|
||||
./weed mini -dir=/data
|
||||
```
|
||||
|
||||
This single command starts a complete SeaweedFS setup with:
|
||||
- **Master UI**: http://localhost:9333
|
||||
- **Volume Server**: http://localhost:9340
|
||||
- **Filer UI**: http://localhost:8888
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
- **WebDAV**: http://localhost:7333
|
||||
- **Admin UI**: http://localhost:23646
|
||||
|
||||
Perfect for development, testing, learning SeaweedFS, and single node deployments!
|
||||
|
||||
## Quick Start for S3 API on Docker ##
|
||||
|
||||
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
|
||||
@@ -111,9 +85,12 @@ Perfect for development, testing, learning SeaweedFS, and single node deployment
|
||||
## Quick Start with Single Binary ##
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
|
||||
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
## Quick Start SeaweedFS S3 on AWS ##
|
||||
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
|
||||
|
||||
# Introduction #
|
||||
|
||||
@@ -229,8 +206,8 @@ SeaweedFS uses HTTP REST operations to read, write, and delete. The responses ar
|
||||
### Start Volume Servers ###
|
||||
|
||||
```
|
||||
> weed volume -dir="/tmp/data1" -max=5 -master="localhost:9333" -port=8080 &
|
||||
> weed volume -dir="/tmp/data2" -max=10 -master="localhost:9333" -port=8081 &
|
||||
> weed volume -dir="/tmp/data1" -max=5 -mserver="localhost:9333" -port=8080 &
|
||||
> weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
|
||||
```
|
||||
|
||||
### Write File ###
|
||||
@@ -543,8 +520,6 @@ cd seaweedfs/weed && make install
|
||||
|
||||
Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
|
||||
|
||||
For more installation options, including how to run with Docker, see the [Getting Started guide](https://github.com/seaweedfs/seaweedfs/wiki/Getting-Started).
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
## Disk Related Topics ##
|
||||
@@ -615,22 +590,65 @@ Percentage of the requests served within a certain time (ms)
|
||||
|
||||
```
|
||||
make benchmark
|
||||
warp: Benchmark data written to "warp-mixed-2025-12-05[194844]-kBpU.csv.zst"
|
||||
|
||||
warp: Benchmark data written to "warp-mixed-2023-10-16[102354]-l70a.csv.zst"
|
||||
Mixed operations.
|
||||
Operation: DELETE, 10%, Concurrency: 20, Ran 42s.
|
||||
* Throughput: 55.13 obj/s
|
||||
Operation: DELETE, 10%, Concurrency: 20, Ran 4m59s.
|
||||
* Throughput: 6.19 obj/s
|
||||
|
||||
Operation: GET, 45%, Concurrency: 20, Ran 42s.
|
||||
* Throughput: 2477.45 MiB/s, 247.75 obj/s
|
||||
Operation: GET, 45%, Concurrency: 20, Ran 5m0s.
|
||||
* Throughput: 279.85 MiB/s, 27.99 obj/s
|
||||
|
||||
Operation: PUT, 15%, Concurrency: 20, Ran 42s.
|
||||
* Throughput: 825.85 MiB/s, 82.59 obj/s
|
||||
Operation: PUT, 15%, Concurrency: 20, Ran 5m0s.
|
||||
* Throughput: 89.86 MiB/s, 8.99 obj/s
|
||||
|
||||
Operation: STAT, 30%, Concurrency: 20, Ran 42s.
|
||||
* Throughput: 165.27 obj/s
|
||||
Operation: STAT, 30%, Concurrency: 20, Ran 5m0s.
|
||||
* Throughput: 18.63 obj/s
|
||||
|
||||
Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
|
||||
Cluster Total: 369.74 MiB/s, 61.79 obj/s, 0 errors over 5m0s.
|
||||
```
|
||||
|
||||
To see segmented request statistics, use the --analyze.v parameter.
|
||||
```
|
||||
warp analyze --analyze.v warp-mixed-2023-10-16[102354]-l70a.csv.zst
|
||||
18642 operations loaded... Done!
|
||||
Mixed operations.
|
||||
----------------------------------------
|
||||
Operation: DELETE - total: 1854, 10.0%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
|
||||
* Throughput: 6.19 obj/s
|
||||
|
||||
Requests considered: 1855:
|
||||
* Avg: 104ms, 50%: 30ms, 90%: 207ms, 99%: 1.355s, Fastest: 1ms, Slowest: 4.613s, StdDev: 320ms
|
||||
|
||||
----------------------------------------
|
||||
Operation: GET - total: 8388, 45.3%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.12 +0500 +05
|
||||
* Throughput: 279.77 MiB/s, 27.98 obj/s
|
||||
|
||||
Requests considered: 8389:
|
||||
* Avg: 221ms, 50%: 106ms, 90%: 492ms, 99%: 1.739s, Fastest: 8ms, Slowest: 8.633s, StdDev: 383ms
|
||||
* TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 171ms, 99th: 669ms, Worst: 4.783s StdDev: 163ms
|
||||
* First Access: Avg: 240ms, 50%: 105ms, 90%: 511ms, 99%: 2.08s, Fastest: 12ms, Slowest: 8.633s, StdDev: 480ms
|
||||
* First Access TTFB: Avg: 88ms, Best: 2ms, 25th: 24ms, Median: 38ms, 75th: 64ms, 90th: 179ms, 99th: 919ms, Worst: 4.783s StdDev: 199ms
|
||||
* Last Access: Avg: 219ms, 50%: 106ms, 90%: 463ms, 99%: 1.782s, Fastest: 9ms, Slowest: 8.633s, StdDev: 416ms
|
||||
* Last Access TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 161ms, 99th: 657ms, Worst: 4.783s StdDev: 176ms
|
||||
|
||||
----------------------------------------
|
||||
Operation: PUT - total: 2688, 14.5%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
|
||||
* Throughput: 89.83 MiB/s, 8.98 obj/s
|
||||
|
||||
Requests considered: 2689:
|
||||
* Avg: 1.165s, 50%: 878ms, 90%: 2.015s, 99%: 5.74s, Fastest: 99ms, Slowest: 8.264s, StdDev: 968ms
|
||||
|
||||
----------------------------------------
|
||||
Operation: STAT - total: 5586, 30.2%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.113 +0500 +05
|
||||
* Throughput: 18.63 obj/s
|
||||
|
||||
Requests considered: 5587:
|
||||
* Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 80ms, Fastest: 0s, Slowest: 245ms, StdDev: 17ms
|
||||
* First Access: Avg: 14ms, 50%: 10ms, 90%: 33ms, 99%: 69ms, Fastest: 0s, Slowest: 203ms, StdDev: 16ms
|
||||
* Last Access: Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 74ms, Fastest: 0s, Slowest: 203ms, StdDev: 17ms
|
||||
|
||||
Cluster Total: 369.64 MiB/s, 61.77 obj/s, 0 errors over 5m0s.
|
||||
Total Errors:0.
|
||||
```
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
# SQL Query Engine Feature, Dev, and Test Plan
|
||||
|
||||
This document outlines the plan for adding SQL querying support to SeaweedFS, focusing on reading and analyzing data from Message Queue (MQ) topics.
|
||||
|
||||
## Feature Plan
|
||||
|
||||
**1. Goal**
|
||||
|
||||
To provide a SQL querying interface for SeaweedFS, enabling analytics on existing MQ topics. This enables:
|
||||
- Basic querying with SELECT, WHERE, aggregations on MQ topics
|
||||
- Schema discovery and metadata operations (SHOW DATABASES, SHOW TABLES, DESCRIBE)
|
||||
- In-place analytics on Parquet-stored messages without data movement
|
||||
|
||||
**2. Key Features**
|
||||
|
||||
* **Schema Discovery and Metadata:**
|
||||
* `SHOW DATABASES` - List all MQ namespaces
|
||||
* `SHOW TABLES` - List all topics in a namespace
|
||||
* `DESCRIBE table_name` - Show topic schema details
|
||||
* Automatic schema detection from existing Parquet data
|
||||
* **Basic Query Engine:**
|
||||
* `SELECT` support with `WHERE`, `LIMIT`, `OFFSET`
|
||||
* Aggregation functions: `COUNT()`, `SUM()`, `AVG()`, `MIN()`, `MAX()`
|
||||
* Temporal queries with timestamp-based filtering
|
||||
* **User Interfaces:**
|
||||
* New CLI command `weed sql` with interactive shell mode
|
||||
* Optional: Web UI for query execution and result visualization
|
||||
* **Output Formats:**
|
||||
* JSON (default), CSV, Parquet for result sets
|
||||
* Streaming results for large queries
|
||||
* Pagination support for result navigation
|
||||
|
||||
## Development Plan
|
||||
|
||||
|
||||
|
||||
**3. Data Source Integration**
|
||||
|
||||
* **MQ Topic Connector (Primary):**
|
||||
* Build on existing `weed/mq/logstore/read_parquet_to_log.go`
|
||||
* Implement efficient Parquet scanning with predicate pushdown
|
||||
* Support schema evolution and backward compatibility
|
||||
* Handle partition-based parallelism for scalable queries
|
||||
* **Schema Registry Integration:**
|
||||
* Extend `weed/mq/schema/schema.go` for SQL metadata operations
|
||||
* Read existing topic schemas for query planning
|
||||
* Handle schema evolution during query execution
|
||||
|
||||
**4. API & CLI Integration**
|
||||
|
||||
* **CLI Command:**
|
||||
* New `weed sql` command with interactive shell mode (similar to `weed shell`)
|
||||
* Support for script execution and result formatting
|
||||
* Connection management for remote SeaweedFS clusters
|
||||
* **gRPC API:**
|
||||
* Add SQL service to existing MQ broker gRPC interface
|
||||
* Enable efficient query execution with streaming results
|
||||
|
||||
## Example Usage Scenarios
|
||||
|
||||
**Scenario 1: Schema Discovery and Metadata**
|
||||
```sql
|
||||
-- List all namespaces (databases)
|
||||
SHOW DATABASES;
|
||||
|
||||
-- List topics in a namespace
|
||||
USE my_namespace;
|
||||
SHOW TABLES;
|
||||
|
||||
-- View topic structure and discovered schema
|
||||
DESCRIBE user_events;
|
||||
```
|
||||
|
||||
**Scenario 2: Data Querying**
|
||||
```sql
|
||||
-- Basic filtering and projection
|
||||
SELECT user_id, event_type, timestamp
|
||||
FROM user_events
|
||||
WHERE timestamp > 1640995200000
|
||||
LIMIT 100;
|
||||
|
||||
-- Aggregation queries
|
||||
SELECT COUNT(*) as event_count
|
||||
FROM user_events
|
||||
WHERE timestamp >= 1640995200000;
|
||||
|
||||
-- More aggregation examples
|
||||
SELECT MAX(timestamp), MIN(timestamp)
|
||||
FROM user_events;
|
||||
```
|
||||
|
||||
**Scenario 3: Analytics & Monitoring**
|
||||
```sql
|
||||
-- Basic analytics
|
||||
SELECT COUNT(*) as total_events
|
||||
FROM user_events
|
||||
WHERE timestamp >= 1640995200000;
|
||||
|
||||
-- Simple monitoring
|
||||
SELECT AVG(response_time) as avg_response
|
||||
FROM api_logs
|
||||
WHERE timestamp >= 1640995200000;
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
SQL Query Flow:
|
||||
1. Parse SQL 2. Plan & Optimize 3. Execute Query
|
||||
┌─────────────┐ ┌──────────────┐ ┌─────────────────┐ ┌──────────────┐
|
||||
│ Client │ │ SQL Parser │ │ Query Planner │ │ Execution │
|
||||
│ (CLI) │──→ │ PostgreSQL │──→ │ & Optimizer │──→ │ Engine │
|
||||
│ │ │ (Custom) │ │ │ │ │
|
||||
└─────────────┘ └──────────────┘ └─────────────────┘ └──────────────┘
|
||||
│ │
|
||||
│ Schema Lookup │ Data Access
|
||||
▼ ▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Schema Catalog │
|
||||
│ • Namespace → Database mapping │
|
||||
│ • Topic → Table mapping │
|
||||
│ • Schema version management │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
▲
|
||||
│ Metadata
|
||||
│
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ MQ Storage Layer │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ▲ │
|
||||
│ │ Topic A │ │ Topic B │ │ Topic C │ │ ... │ │ │
|
||||
│ │ (Parquet) │ │ (Parquet) │ │ (Parquet) │ │ (Parquet) │ │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ │ │
|
||||
└──────────────────────────────────────────────────────────────────────────│──┘
|
||||
│
|
||||
Data Access
|
||||
```
|
||||
|
||||
|
||||
## Success Metrics
|
||||
|
||||
* **Feature Completeness:** Support for all specified SELECT operations and metadata commands
|
||||
* **Performance:**
|
||||
* **Simple SELECT queries**: < 100ms latency for single-table queries with up to 3 WHERE predicates on ≤ 100K records
|
||||
* **Complex queries**: < 1s latency for queries involving aggregations (COUNT, SUM, MAX, MIN) on ≤ 1M records
|
||||
* **Time-range queries**: < 500ms for timestamp-based filtering on ≤ 500K records within 24-hour windows
|
||||
* **Scalability:** Handle topics with millions of messages efficiently
|
||||
@@ -0,0 +1,169 @@
|
||||
# Server-Side Encryption with Customer-Provided Keys (SSE-C) Implementation
|
||||
|
||||
This document describes the implementation of SSE-C support in SeaweedFS, addressing the feature request from [GitHub Discussion #5361](https://github.com/seaweedfs/seaweedfs/discussions/5361).
|
||||
|
||||
## Overview
|
||||
|
||||
SSE-C allows clients to provide their own encryption keys for server-side encryption of objects stored in SeaweedFS. The server encrypts the data using the customer-provided AES-256 key but does not store the key itself - only an MD5 hash of the key for validation purposes.
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Architecture
|
||||
|
||||
The SSE-C implementation follows a transparent encryption/decryption pattern:
|
||||
|
||||
1. **Upload (PUT/POST)**: Data is encrypted with the customer key before being stored
|
||||
2. **Download (GET/HEAD)**: Encrypted data is decrypted on-the-fly using the customer key
|
||||
3. **Metadata Storage**: Only the encryption algorithm and key MD5 are stored as metadata
|
||||
|
||||
### Key Components
|
||||
|
||||
#### 1. Constants and Headers (`weed/s3api/s3_constants/header.go`)
|
||||
- Added AWS-compatible SSE-C header constants
|
||||
- Support for both regular and copy-source SSE-C headers
|
||||
|
||||
#### 2. Core SSE-C Logic (`weed/s3api/s3_sse_c.go`)
|
||||
- **SSECustomerKey**: Structure to hold customer encryption key and metadata
|
||||
- **SSECEncryptedReader**: Streaming encryption with AES-256-CTR mode
|
||||
- **SSECDecryptedReader**: Streaming decryption with IV extraction
|
||||
- **validateAndParseSSECHeaders**: Shared validation logic (DRY principle)
|
||||
- **ParseSSECHeaders**: Parse regular SSE-C headers
|
||||
- **ParseSSECCopySourceHeaders**: Parse copy-source SSE-C headers
|
||||
- Header validation and parsing functions
|
||||
- Metadata extraction and response handling
|
||||
|
||||
#### 3. Error Handling (`weed/s3api/s3err/s3api_errors.go`)
|
||||
- New error codes for SSE-C validation failures
|
||||
- AWS-compatible error messages and HTTP status codes
|
||||
|
||||
#### 4. S3 API Integration
|
||||
- **PUT Object Handler**: Encrypts data streams transparently
|
||||
- **GET Object Handler**: Decrypts data streams transparently
|
||||
- **HEAD Object Handler**: Validates keys and returns appropriate headers
|
||||
- **Metadata Storage**: Integrates with existing `SaveAmzMetaData` function
|
||||
|
||||
### Encryption Scheme
|
||||
|
||||
- **Algorithm**: AES-256-CTR (Counter mode)
|
||||
- **Key Size**: 256 bits (32 bytes)
|
||||
- **IV Generation**: Random 16-byte IV per object
|
||||
- **Storage Format**: `[IV][EncryptedData]` where IV is prepended to encrypted content
|
||||
|
||||
### Metadata Storage
|
||||
|
||||
SSE-C metadata is stored in the filer's extended attributes:
|
||||
```
|
||||
x-amz-server-side-encryption-customer-algorithm: "AES256"
|
||||
x-amz-server-side-encryption-customer-key-md5: "<md5-hash-of-key>"
|
||||
```
|
||||
|
||||
## API Compatibility
|
||||
|
||||
### Required Headers for Encryption (PUT/POST)
|
||||
```
|
||||
x-amz-server-side-encryption-customer-algorithm: AES256
|
||||
x-amz-server-side-encryption-customer-key: <base64-encoded-256-bit-key>
|
||||
x-amz-server-side-encryption-customer-key-md5: <md5-hash-of-key>
|
||||
```
|
||||
|
||||
### Required Headers for Decryption (GET/HEAD)
|
||||
Same headers as encryption - the server validates the key MD5 matches.
|
||||
|
||||
### Copy Operations
|
||||
Support for copy-source SSE-C headers:
|
||||
```
|
||||
x-amz-copy-source-server-side-encryption-customer-algorithm
|
||||
x-amz-copy-source-server-side-encryption-customer-key
|
||||
x-amz-copy-source-server-side-encryption-customer-key-md5
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The implementation provides AWS-compatible error responses:
|
||||
|
||||
- **InvalidEncryptionAlgorithmError**: Non-AES256 algorithm specified
|
||||
- **InvalidArgument**: Invalid key format, size, or MD5 mismatch
|
||||
- **Missing customer key**: Object encrypted but no key provided
|
||||
- **Unnecessary customer key**: Object not encrypted but key provided
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **Key Management**: Customer keys are never stored - only MD5 hashes for validation
|
||||
2. **IV Randomness**: Fresh random IV generated for each object
|
||||
3. **Transparent Security**: Volume servers never see unencrypted data
|
||||
4. **Key Validation**: Strict validation of key format, size, and MD5
|
||||
|
||||
## Testing
|
||||
|
||||
Comprehensive test suite covers:
|
||||
- Header validation and parsing (regular and copy-source)
|
||||
- Encryption/decryption round-trip
|
||||
- Error condition handling
|
||||
- Metadata extraction
|
||||
- Code reuse validation (DRY principle)
|
||||
- AWS S3 compatibility
|
||||
|
||||
Run tests with:
|
||||
```bash
|
||||
go test -v ./weed/s3api
|
||||
|
||||
## Usage Example
|
||||
|
||||
### Upload with SSE-C
|
||||
```bash
|
||||
# Generate a 256-bit key
|
||||
KEY=$(openssl rand -base64 32)
|
||||
KEY_MD5=$(echo -n "$KEY" | base64 -d | openssl dgst -md5 -binary | base64)
|
||||
|
||||
# Upload object with SSE-C
|
||||
curl -X PUT "http://localhost:8333/bucket/object" \
|
||||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||||
-H "x-amz-server-side-encryption-customer-key-md5: $KEY_MD5" \
|
||||
--data-binary @file.txt
|
||||
```
|
||||
|
||||
### Download with SSE-C
|
||||
```bash
|
||||
# Download object with SSE-C (same key required)
|
||||
curl "http://localhost:8333/bucket/object" \
|
||||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||||
-H "x-amz-server-side-encryption-customer-key-md5: $KEY_MD5"
|
||||
```
|
||||
|
||||
## Integration Points
|
||||
|
||||
### Existing SeaweedFS Features
|
||||
- **Filer Metadata**: Extends existing metadata storage
|
||||
- **Volume Servers**: No changes required - store encrypted data transparently
|
||||
- **S3 API**: Integrates seamlessly with existing handlers
|
||||
- **Versioning**: Compatible with object versioning
|
||||
- **Multipart Upload**: Ready for multipart upload integration
|
||||
|
||||
### Future Enhancements
|
||||
- **SSE-S3**: Server-managed encryption keys
|
||||
- **SSE-KMS**: External key management service integration
|
||||
- **Performance Optimization**: Hardware acceleration for encryption
|
||||
- **Compliance**: Enhanced audit logging for encrypted objects
|
||||
|
||||
## File Changes Summary
|
||||
|
||||
1. **`weed/s3api/s3_constants/header.go`** - Added SSE-C header constants
|
||||
2. **`weed/s3api/s3_sse_c.go`** - Core SSE-C implementation (NEW)
|
||||
3. **`weed/s3api/s3_sse_c_test.go`** - Comprehensive test suite (NEW)
|
||||
4. **`weed/s3api/s3err/s3api_errors.go`** - Added SSE-C error codes
|
||||
5. **`weed/s3api/s3api_object_handlers.go`** - GET/HEAD with SSE-C support
|
||||
6. **`weed/s3api/s3api_object_handlers_put.go`** - PUT with SSE-C support
|
||||
7. **`weed/server/filer_server_handlers_write_autochunk.go`** - Metadata storage
|
||||
|
||||
## Compliance
|
||||
|
||||
This implementation follows the [AWS S3 SSE-C specification](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) for maximum compatibility with existing S3 clients and tools.
|
||||
|
||||
## Performance Impact
|
||||
|
||||
- **Encryption Overhead**: Minimal CPU impact with efficient AES-CTR streaming
|
||||
- **Memory Usage**: Constant memory usage via streaming encryption/decryption
|
||||
- **Storage Overhead**: 16 bytes per object for IV storage
|
||||
- **Network**: No additional network overhead
|
||||
+2
-18
@@ -2,28 +2,12 @@ FROM ubuntu:22.04
|
||||
|
||||
LABEL author="Chris Lu"
|
||||
|
||||
# Use faster mirrors and optimize package installation
|
||||
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
|
||||
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
|
||||
# For testing purposes, running as root avoids permission complexities and dependency
|
||||
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
curl \
|
||||
fio \
|
||||
fuse \
|
||||
ca-certificates \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& rm -rf /tmp/* \
|
||||
&& rm -rf /var/tmp/*
|
||||
RUN apt-get update && apt-get install -y curl fio fuse
|
||||
RUN mkdir -p /etc/seaweedfs /data/filerldb2
|
||||
|
||||
COPY ./weed /usr/bin/
|
||||
COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint_e2e.sh /entrypoint.sh
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# volume server grpc port
|
||||
EXPOSE 18080
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
FROM golang:1.24 AS builder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y build-essential wget ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ARG FDB_VERSION=7.4.5
|
||||
ENV FDB_VERSION=${FDB_VERSION}
|
||||
ARG TARGETARCH
|
||||
|
||||
# Install FoundationDB client libraries with SHA256 checksum verification
|
||||
# Known SHA256 checksums for FoundationDB client packages (verified 2025-01-19)
|
||||
# To add checksums for new versions: run docker/get_fdb_checksum.sh <version> <arch>
|
||||
RUN cd /tmp && \
|
||||
case "${TARGETARCH}" in \
|
||||
"amd64") FDB_ARCH="amd64"; PACKAGE_ARCH="amd64" ;; \
|
||||
"arm64") FDB_ARCH="arm64"; PACKAGE_ARCH="aarch64" ;; \
|
||||
*) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac && \
|
||||
case "${FDB_VERSION}_${FDB_ARCH}" in \
|
||||
"7.4.5_amd64") \
|
||||
EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;; \
|
||||
"7.4.5_arm64") \
|
||||
EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
|
||||
"7.3.43_amd64") \
|
||||
EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;; \
|
||||
"7.3.43_arm64") \
|
||||
echo "ERROR: FoundationDB ${FDB_VERSION} does not publish arm64 client packages." >&2; \
|
||||
echo "Please upgrade to 7.4.5+ when targeting arm64." >&2; \
|
||||
exit 1 ;; \
|
||||
*) \
|
||||
echo "ERROR: No checksum available for FDB version ${FDB_VERSION} on ${FDB_ARCH}" >&2; \
|
||||
echo "This is a security requirement. To add verification:" >&2; \
|
||||
echo " 1. Run: docker/get_fdb_checksum.sh ${FDB_VERSION} ${FDB_ARCH}" >&2; \
|
||||
echo " 2. Add the checksum to this Dockerfile" >&2; \
|
||||
echo "Refusing to proceed without checksum verification." >&2; \
|
||||
exit 1 ;; \
|
||||
esac && \
|
||||
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb" && \
|
||||
wget --timeout=30 --tries=3 https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE} && \
|
||||
echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c - || \
|
||||
(echo "ERROR: Checksum verification failed for FoundationDB ${FDB_VERSION} (${FDB_ARCH})" >&2; \
|
||||
echo "Expected: ${EXPECTED_SHA256}" >&2; \
|
||||
echo "This indicates either a corrupted download or potential tampering." >&2; \
|
||||
exit 1) && \
|
||||
dpkg -i ${PACKAGE} && \
|
||||
rm ${PACKAGE}
|
||||
|
||||
# Set up FoundationDB environment variables for CGO
|
||||
ENV CGO_CFLAGS="-I/usr/include/foundationdb"
|
||||
ENV CGO_LDFLAGS="-lfdb_c"
|
||||
|
||||
# build SeaweedFS sources; prefer local context but fall back to git clone if context only has docker files
|
||||
ARG SOURCE_REF=master
|
||||
WORKDIR /go/src/github.com/seaweedfs/seaweedfs
|
||||
COPY . .
|
||||
RUN set -euo pipefail && \
|
||||
if [ ! -d weed ]; then \
|
||||
echo "Local build context does not include SeaweedFS sources; cloning ${SOURCE_REF}" >&2; \
|
||||
mkdir -p /tmp/local-context && cp -a /go/src/github.com/seaweedfs/seaweedfs/. /tmp/local-context && \
|
||||
cd / && rm -rf /go/src/github.com/seaweedfs/seaweedfs && \
|
||||
git clone --depth 1 --branch ${SOURCE_REF} https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs && \
|
||||
cp -a /tmp/local-context/. /go/src/github.com/seaweedfs/seaweedfs/docker/ && \
|
||||
rm -rf /tmp/local-context && \
|
||||
cd /go/src/github.com/seaweedfs/seaweedfs; \
|
||||
fi && \
|
||||
cd weed \
|
||||
&& COMMIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${COMMIT_SHA}" \
|
||||
&& go install -tags "5BytesOffset foundationdb" -ldflags "${LDFLAGS}"
|
||||
|
||||
|
||||
FROM debian:bookworm-slim AS final
|
||||
LABEL author="Chris Lu"
|
||||
|
||||
# Install runtime dependencies first
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
fuse \
|
||||
wget && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Reuse FoundationDB artifacts installed during the build stage
|
||||
COPY --from=builder /usr/lib/libfdb_c* /usr/lib/
|
||||
COPY --from=builder /usr/lib/foundationdb /usr/lib/foundationdb
|
||||
COPY --from=builder /usr/bin/fdb* /usr/bin/
|
||||
RUN ldconfig
|
||||
|
||||
# Copy SeaweedFS binary and configuration
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_foundationdb.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Create non-root user
|
||||
RUN groupadd -g 1000 seaweed && \
|
||||
useradd -u 1000 -g seaweed -s /bin/bash -m seaweed
|
||||
|
||||
# volume server gprc port
|
||||
EXPOSE 18080
|
||||
# volume server http port
|
||||
EXPOSE 8080
|
||||
# filer server gprc port
|
||||
EXPOSE 18888
|
||||
# filer server http port
|
||||
EXPOSE 8888
|
||||
# master server shared gprc port
|
||||
EXPOSE 19333
|
||||
# master server shared http port
|
||||
EXPOSE 9333
|
||||
# s3 server http port
|
||||
EXPOSE 8333
|
||||
# webdav server http port
|
||||
EXPOSE 7333
|
||||
|
||||
# Create data directory and set proper ownership for seaweed user
|
||||
RUN mkdir -p /data && \
|
||||
chown -R seaweed:seaweed /data && \
|
||||
chown -R seaweed:seaweed /etc/seaweedfs && \
|
||||
chmod 755 /entrypoint.sh
|
||||
|
||||
VOLUME /data
|
||||
|
||||
WORKDIR /data
|
||||
|
||||
# Switch to non-root user
|
||||
USER seaweed
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
FROM golang:1.24-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
# Clone with full history and all tags to ensure all commits are available
|
||||
RUN git clone --no-single-branch --tags https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG TAGS
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
|
||||
(git checkout $BRANCH || \
|
||||
(echo "Checkout failed, fetching all history..." && \
|
||||
git fetch --all --tags --prune && \
|
||||
git checkout $BRANCH) || \
|
||||
(echo "ERROR: Branch/commit $BRANCH not found in repository" && \
|
||||
echo "Available branches:" && git branch -a && exit 1))
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
@@ -22,14 +15,7 @@ COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# FIPS 140-3 mode is ON by default (Go 1.24+)
|
||||
# To disable: docker run -e GODEBUG=fips140=off ...
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
RUN apk add fuse # for weed mount
|
||||
|
||||
# volume server gprc port
|
||||
EXPOSE 18080
|
||||
@@ -48,14 +34,11 @@ EXPOSE 8333
|
||||
# webdav server http port
|
||||
EXPOSE 7333
|
||||
|
||||
# Create data directory and set proper ownership for seaweed user
|
||||
RUN mkdir -p /data/filerldb2 && \
|
||||
chown -R seaweed:seaweed /data && \
|
||||
chown -R seaweed:seaweed /etc/seaweedfs && \
|
||||
chmod 755 /entrypoint.sh
|
||||
RUN mkdir -p /data/filerldb2
|
||||
|
||||
VOLUME /data
|
||||
WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
+8
-13
@@ -1,15 +1,13 @@
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY ./weed /usr/bin/weed
|
||||
RUN chmod +x /usr/bin/weed && ls -la /usr/bin/weed
|
||||
COPY ./weed /usr/bin/
|
||||
COPY ./weed_pub* /usr/bin/
|
||||
COPY ./weed_sub* /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
RUN apk add fuse # for weed mount
|
||||
RUN apk add curl # for health checks
|
||||
|
||||
# volume server grpc port
|
||||
EXPOSE 18080
|
||||
@@ -28,14 +26,11 @@ EXPOSE 8333
|
||||
# webdav server http port
|
||||
EXPOSE 7333
|
||||
|
||||
# Create data directory and set proper ownership for seaweed user
|
||||
RUN mkdir -p /data/filerldb2 && \
|
||||
chown -R seaweed:seaweed /data && \
|
||||
chown -R seaweed:seaweed /etc/seaweedfs && \
|
||||
chmod 755 /entrypoint.sh
|
||||
RUN mkdir -p /data/filerldb2
|
||||
|
||||
VOLUME /data
|
||||
WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
FROM golang:1.24 AS builder
|
||||
FROM golang:1.24 as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ARG ROCKSDB_VERSION=v10.5.1
|
||||
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
|
||||
ENV ROCKSDB_VERSION v10.2.1
|
||||
|
||||
# build RocksDB
|
||||
RUN cd /tmp && \
|
||||
git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
|
||||
cd rocksdb && \
|
||||
PORTABLE=1 make -j"$(nproc)" static_lib && \
|
||||
PORTABLE=1 make static_lib && \
|
||||
make install-static
|
||||
|
||||
ENV CGO_CFLAGS="-I/tmp/rocksdb/include"
|
||||
ENV CGO_LDFLAGS="-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
|
||||
ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
|
||||
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
|
||||
|
||||
@@ -1,25 +1,24 @@
|
||||
FROM golang:1.24 AS builder
|
||||
FROM golang:1.24 as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ARG ROCKSDB_VERSION=v10.5.1
|
||||
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
|
||||
ENV ROCKSDB_VERSION v10.2.1
|
||||
|
||||
# build RocksDB
|
||||
RUN cd /tmp && \
|
||||
git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
|
||||
cd rocksdb && \
|
||||
PORTABLE=1 make -j"$(nproc)" static_lib && \
|
||||
PORTABLE=1 make static_lib && \
|
||||
make install-static
|
||||
|
||||
ENV CGO_CFLAGS="-I/tmp/rocksdb/include"
|
||||
ENV CGO_LDFLAGS="-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
|
||||
ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
|
||||
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
|
||||
|
||||
# build SeaweedFS
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG BRANCH=master
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
@@ -32,11 +31,7 @@ COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
RUN apk add fuse snappy gflags
|
||||
|
||||
# volume server gprc port
|
||||
EXPOSE 18080
|
||||
@@ -55,15 +50,12 @@ EXPOSE 8333
|
||||
# webdav server http port
|
||||
EXPOSE 7333
|
||||
|
||||
# Create data directory and set proper ownership for seaweed user
|
||||
RUN mkdir -p /data/filer_rocksdb && \
|
||||
chown -R seaweed:seaweed /data && \
|
||||
chown -R seaweed:seaweed /etc/seaweedfs && \
|
||||
chmod 755 /entrypoint.sh
|
||||
RUN mkdir -p /data/filer_rocksdb
|
||||
|
||||
VOLUME /data
|
||||
|
||||
WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
@@ -15,11 +15,7 @@ COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
RUN apk add fuse snappy gflags tmux
|
||||
|
||||
# volume server gprc port
|
||||
EXPOSE 18080
|
||||
@@ -38,15 +34,12 @@ EXPOSE 8333
|
||||
# webdav server http port
|
||||
EXPOSE 7333
|
||||
|
||||
# Create data directory and set proper ownership for seaweed user
|
||||
RUN mkdir -p /data/filer_rocksdb && \
|
||||
chown -R seaweed:seaweed /data && \
|
||||
chown -R seaweed:seaweed /etc/seaweedfs && \
|
||||
chmod 755 /entrypoint.sh
|
||||
RUN mkdir -p /data/filer_rocksdb
|
||||
|
||||
VOLUME /data
|
||||
|
||||
WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
+2
-14
@@ -5,34 +5,22 @@ all: gen
|
||||
gen: dev
|
||||
|
||||
cgo ?= 0
|
||||
ldflags_extra ?= -extldflags -static
|
||||
|
||||
binary:
|
||||
export SWCOMMIT=$(shell git rev-parse --short HEAD)
|
||||
export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(SWCOMMIT)"
|
||||
cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w $(ldflags_extra) $(SWLDFLAGS)" -o weed_binary && mv weed_binary ../docker/weed
|
||||
cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" -o weed_binary && mv weed_binary ../docker/weed
|
||||
cd ../other/mq_client_example/agent_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_pub_record ../../../docker/
|
||||
cd ../other/mq_client_example/agent_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_sub_record ../../../docker/
|
||||
|
||||
# Race detector requires CGO and dynamic linking - don't use -static
|
||||
binary_race: options = -race
|
||||
binary_race: cgo = 1
|
||||
binary_race: ldflags_extra =
|
||||
binary_race: binary
|
||||
|
||||
build: binary
|
||||
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
|
||||
|
||||
build_e2e: binary_race
|
||||
docker buildx build \
|
||||
--cache-from=type=local,src=/tmp/.buildx-cache \
|
||||
--cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \
|
||||
--load \
|
||||
-t chrislusf/seaweedfs:e2e \
|
||||
-f Dockerfile.e2e .
|
||||
# Move cache to avoid growing cache size
|
||||
rm -rf /tmp/.buildx-cache || true
|
||||
mv /tmp/.buildx-cache-new /tmp/.buildx-cache || true
|
||||
docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
|
||||
|
||||
go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset,tarantool
|
||||
docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build .
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SeaweedFS Admin Integration Test Makefile
|
||||
# Tests the admin server and worker functionality using official weed commands
|
||||
|
||||
.PHONY: help build build-and-restart restart-workers start stop restart logs clean status test admin-ui worker-logs master-logs admin-logs vacuum-test vacuum-demo vacuum-status vacuum-data vacuum-data-high vacuum-data-low vacuum-continuous vacuum-clean vacuum-help
|
||||
.PHONY: help build build-and-restart restart-workers restart-admin start stop restart logs clean status test admin-ui worker-logs master-logs admin-logs vacuum-test vacuum-demo vacuum-status vacuum-data vacuum-data-high vacuum-data-low vacuum-continuous vacuum-clean vacuum-help file-generation-test file-deletion-test all-file-tests
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
COMPOSE_FILE := docker-compose-ec-test.yml
|
||||
@@ -34,29 +34,45 @@ restart-workers: ## Restart all workers to reconnect to admin server
|
||||
@docker-compose -f $(COMPOSE_FILE) restart worker1 worker2 worker3
|
||||
@echo "✅ Workers restarted and will reconnect to admin server"
|
||||
|
||||
restart-admin: ## Restart admin server (useful after deadlock fixes)
|
||||
@echo "🔄 Restarting admin server..."
|
||||
@docker-compose -f $(COMPOSE_FILE) restart admin
|
||||
@echo "✅ Admin server restarted"
|
||||
@echo "🌐 Admin UI: http://localhost:23646/"
|
||||
|
||||
help: ## Show this help message
|
||||
@echo "SeaweedFS Admin Integration Test"
|
||||
@echo "================================"
|
||||
@echo "Tests admin server task distribution to workers using official weed commands"
|
||||
@echo ""
|
||||
@echo "🏗️ Cluster Management:"
|
||||
@grep -E '^(start|stop|restart|clean|status|build):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
|
||||
@grep -E '^(start|stop|restart|restart-admin|restart-workers|clean|status|build):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
|
||||
@echo ""
|
||||
@echo "🧪 Testing:"
|
||||
@grep -E '^(test|demo|validate|quick-test):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
|
||||
@grep -E '^(test|demo|validate|quick-test|file-.*-test|all-file-tests):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
|
||||
@echo ""
|
||||
@echo "🗑️ Vacuum Testing:"
|
||||
@grep -E '^vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
|
||||
@echo ""
|
||||
@echo "🔧 EC Vacuum Testing:"
|
||||
@grep -E '^ec-vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
|
||||
@echo ""
|
||||
@echo "📜 Monitoring:"
|
||||
@grep -E '^(logs|admin-logs|worker-logs|master-logs|admin-ui):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
|
||||
@echo ""
|
||||
@echo "🚀 Quick Start:"
|
||||
@echo " make start # Start cluster"
|
||||
@echo " make vacuum-test # Test vacuum tasks"
|
||||
@echo " make vacuum-help # Vacuum testing guide"
|
||||
@echo " make vacuum-test # Test regular vacuum tasks"
|
||||
@echo " make ec-vacuum-test # Test EC vacuum tasks"
|
||||
@echo " make file-generation-test # Test file generation (600 files)"
|
||||
@echo " make file-deletion-test # Test file deletion (300 files)"
|
||||
@echo " make all-file-tests # Run both file tests"
|
||||
@echo " make vacuum-help # Regular vacuum guide"
|
||||
@echo " make ec-vacuum-help # EC vacuum guide"
|
||||
@echo ""
|
||||
@echo "💡 For detailed vacuum testing: make vacuum-help"
|
||||
@echo "💡 For detailed testing guides:"
|
||||
@echo " make vacuum-help # Regular vacuum testing"
|
||||
@echo " make ec-vacuum-help # EC vacuum testing"
|
||||
|
||||
start: ## Start the complete SeaweedFS cluster with admin and workers
|
||||
@echo "🚀 Starting SeaweedFS cluster with admin and workers..."
|
||||
@@ -316,6 +332,190 @@ vacuum-clean: ## Clean up vacuum test data (removes all volumes!)
|
||||
@docker-compose -f $(COMPOSE_FILE) up -d
|
||||
@echo "✅ Clean up complete. Fresh volumes ready for testing."
|
||||
|
||||
# EC Vacuum Testing Targets
|
||||
file-generation-test: ## Run the file generation test (600 files of 100KB to volume 1)
|
||||
@echo "🧪 Running File Generation Test"
|
||||
@echo "==============================="
|
||||
@echo "1️⃣ Ensuring cluster is running..."
|
||||
@docker-compose -f $(COMPOSE_FILE) up -d
|
||||
@echo "2️⃣ Waiting for cluster to be ready..."
|
||||
@sleep 10
|
||||
@echo "3️⃣ Running file generation test..."
|
||||
@go test -v . -run TestFileGeneration
|
||||
@echo "✅ File generation test completed!"
|
||||
@echo "💡 This test generates 600 files of 100KB each to volume 1 with hardcoded cookie"
|
||||
|
||||
file-deletion-test: ## Run the file deletion test (delete 300 files from volume 1)
|
||||
@echo "🧪 Running File Deletion Test"
|
||||
@echo "============================="
|
||||
@echo "1️⃣ Ensuring cluster is running..."
|
||||
@docker-compose -f $(COMPOSE_FILE) up -d
|
||||
@echo "2️⃣ Waiting for cluster to be ready..."
|
||||
@sleep 10
|
||||
@echo "3️⃣ Running file deletion test..."
|
||||
@go test -v . -run TestFileDeletion
|
||||
@echo "✅ File deletion test completed!"
|
||||
@echo "💡 This test generates 600 files then deletes exactly 300 of them"
|
||||
|
||||
all-file-tests: ## Run both file generation and deletion tests
|
||||
@echo "🧪 Running All File Tests"
|
||||
@echo "========================="
|
||||
@echo "1️⃣ Ensuring cluster is running..."
|
||||
@docker-compose -f $(COMPOSE_FILE) up -d
|
||||
@echo "2️⃣ Waiting for cluster to be ready..."
|
||||
@sleep 10
|
||||
@echo "3️⃣ Running file generation test..."
|
||||
@go test -v . -run TestFileGeneration
|
||||
@echo "4️⃣ Running file deletion test..."
|
||||
@go test -v . -run TestFileDeletion
|
||||
@echo "✅ All file tests completed!"
|
||||
|
||||
ec-vacuum-go-test: ## Run the Go-based EC vacuum integration test with detailed file ID tracking (legacy)
|
||||
@echo "🧪 Running EC Vacuum Go Integration Test"
|
||||
@echo "========================================"
|
||||
@echo "1️⃣ Ensuring cluster is running..."
|
||||
@docker-compose -f $(COMPOSE_FILE) up -d
|
||||
@echo "2️⃣ Waiting for cluster to be ready..."
|
||||
@sleep 10
|
||||
@echo "3️⃣ Running Go test with file ID tracking..."
|
||||
@go test -v . -run TestECVolumeVacuum
|
||||
@echo "✅ EC Vacuum Go test completed!"
|
||||
@echo "💡 This test shows which file IDs are written and deleted"
|
||||
|
||||
ec-vacuum-test: ## Generate EC volumes and test EC vacuum functionality
|
||||
@echo "🧪 SeaweedFS EC Vacuum Task Testing"
|
||||
@echo "===================================="
|
||||
@echo ""
|
||||
@echo "1️⃣ Checking cluster health..."
|
||||
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master ready" || (echo "❌ Master not ready. Run 'make start' first." && exit 1)
|
||||
@curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin ready" || (echo "❌ Admin not ready. Run 'make start' first." && exit 1)
|
||||
@echo ""
|
||||
@echo "2️⃣ Generating data to trigger EC encoding..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=30 -size=3000
|
||||
@echo ""
|
||||
@echo "3️⃣ Waiting for EC encoding to complete..."
|
||||
@echo "⏳ This may take 2-3 minutes..."
|
||||
@sleep 120
|
||||
@echo ""
|
||||
@echo "4️⃣ Generating deletions on EC volumes..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.4
|
||||
@echo ""
|
||||
@echo "5️⃣ Configuration Instructions:"
|
||||
@echo " Visit: http://localhost:23646/maintenance/config/ec_vacuum"
|
||||
@echo " Set for testing:"
|
||||
@echo " • Enable EC Vacuum Tasks: ✅ Checked"
|
||||
@echo " • Garbage Threshold: 0.30 (30%)"
|
||||
@echo " • Scan Interval: [60] [Seconds]"
|
||||
@echo " • Min Volume Age: [2] [Minutes]"
|
||||
@echo " • Max Concurrent: 2"
|
||||
@echo ""
|
||||
@echo "6️⃣ Monitor EC vacuum tasks at: http://localhost:23646/maintenance"
|
||||
@echo ""
|
||||
@echo "💡 Use 'make ec-vacuum-status' to check EC volume garbage ratios"
|
||||
|
||||
ec-vacuum-generate: ## Generate large files to trigger EC encoding
|
||||
@echo "📁 Generating data to trigger EC encoding..."
|
||||
@echo "Creating large files targeting >50MB per volume..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=25 -size=3000
|
||||
@echo ""
|
||||
@echo "⏳ Wait 2-3 minutes for EC encoding, then run 'make ec-vacuum-delete'"
|
||||
|
||||
ec-vacuum-delete: ## Create deletions on EC volumes to generate garbage
|
||||
@echo "🗑️ Creating deletions on EC volumes..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=$${DELETE:-0.4}
|
||||
@echo ""
|
||||
@echo "💡 Use 'make ec-vacuum-status' to check garbage ratios"
|
||||
|
||||
ec-vacuum-status: ## Check EC volume status and garbage ratios
|
||||
@echo "📊 EC Volume Status and Garbage Ratios"
|
||||
@echo "====================================="
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=status
|
||||
|
||||
ec-vacuum-continuous: ## Generate continuous EC garbage for testing
|
||||
@echo "🔄 Generating continuous EC garbage for vacuum testing..."
|
||||
@echo "Running 3 rounds with 60-second intervals..."
|
||||
@for i in $$(seq 1 3); do \
|
||||
echo "Round $$i: Generating large files..."; \
|
||||
docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=15 -size=4000; \
|
||||
echo "Waiting 90 seconds for EC encoding..."; \
|
||||
sleep 90; \
|
||||
echo "Creating deletions..."; \
|
||||
docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.5; \
|
||||
echo "Waiting 60 seconds before next round..."; \
|
||||
sleep 60; \
|
||||
done
|
||||
@echo "✅ Continuous EC vacuum test complete. Monitor admin UI for ec_vacuum tasks!"
|
||||
|
||||
ec-vacuum-high: ## Create high garbage on EC volumes (should trigger EC vacuum)
|
||||
@echo "📁 Creating high garbage EC volumes (60% garbage)..."
|
||||
@echo "1. Generating files for EC..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=20 -size=4000
|
||||
@echo "2. Waiting for EC encoding..."
|
||||
@sleep 120
|
||||
@echo "3. Creating high garbage ratio..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.6
|
||||
|
||||
ec-vacuum-low: ## Create low garbage on EC volumes (should NOT trigger EC vacuum)
|
||||
@echo "📁 Creating low garbage EC volumes (20% garbage)..."
|
||||
@echo "1. Generating files for EC..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=generate -files=20 -size=4000
|
||||
@echo "2. Waiting for EC encoding..."
|
||||
@sleep 120
|
||||
@echo "3. Creating low garbage ratio..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=delete -delete=0.2
|
||||
|
||||
ec-vacuum-monitor: ## Monitor EC vacuum task activity in real-time
|
||||
@echo "📊 Monitoring EC Vacuum Task Activity"
|
||||
@echo "===================================="
|
||||
@echo "Press Ctrl+C to stop monitoring"
|
||||
@echo ""
|
||||
@while true; do \
|
||||
echo "=== $(date) ==="; \
|
||||
docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -phase=status; \
|
||||
echo ""; \
|
||||
echo "🔍 Recent admin logs (EC vacuum activity):"; \
|
||||
docker-compose -f $(COMPOSE_FILE) logs --tail=5 admin | grep -i "ec_vacuum\|vacuum.*ec" || echo "No recent EC vacuum activity"; \
|
||||
echo ""; \
|
||||
sleep 30; \
|
||||
done
|
||||
|
||||
ec-vacuum-help: ## Show EC vacuum testing help and examples
|
||||
@echo "🧪 EC Vacuum Testing Commands"
|
||||
@echo "============================="
|
||||
@echo ""
|
||||
@echo "Quick Start:"
|
||||
@echo " make start # Start SeaweedFS cluster"
|
||||
@echo " make ec-vacuum-test # Full EC vacuum test cycle"
|
||||
@echo " make ec-vacuum-status # Check EC volume status"
|
||||
@echo ""
|
||||
@echo "Manual Testing:"
|
||||
@echo " make ec-vacuum-generate # 1. Generate data → trigger EC"
|
||||
@echo " # Wait 2-3 minutes for EC encoding to complete"
|
||||
@echo " make ec-vacuum-delete # 2. Create deletions → garbage"
|
||||
@echo " make ec-vacuum-status # 3. Check garbage ratios"
|
||||
@echo ""
|
||||
@echo "Automated Testing:"
|
||||
@echo " make ec-vacuum-high # High garbage (should trigger)"
|
||||
@echo " make ec-vacuum-low # Low garbage (should NOT trigger)"
|
||||
@echo " make ec-vacuum-continuous # Continuous testing cycle"
|
||||
@echo ""
|
||||
@echo "Monitoring:"
|
||||
@echo " make ec-vacuum-status # Quick EC volume status"
|
||||
@echo " make ec-vacuum-monitor # Real-time monitoring"
|
||||
@echo ""
|
||||
@echo "Configuration:"
|
||||
@echo " Visit: http://localhost:23646/maintenance/config/ec_vacuum"
|
||||
@echo " Monitor: http://localhost:23646/maintenance"
|
||||
@echo ""
|
||||
@echo "💡 EC volumes need time to encode after data generation"
|
||||
@echo "💡 Wait 2-3 minutes between generate and delete phases"
|
||||
@echo ""
|
||||
@echo "Understanding EC Vacuum:"
|
||||
@echo " • Regular volumes → EC volumes (when >50MB)"
|
||||
@echo " • EC vacuum cleans garbage from EC volumes"
|
||||
@echo " • Requires different thresholds than regular vacuum"
|
||||
@echo " • More complex due to shard distribution"
|
||||
|
||||
vacuum-help: ## Show vacuum testing help and examples
|
||||
@echo "🧪 Vacuum Testing Commands (Docker-based)"
|
||||
@echo "=========================================="
|
||||
|
||||
@@ -9,29 +9,33 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
master = flag.String("master", "master:9333", "SeaweedFS master server address")
|
||||
filer = flag.String("filer", "filer1:8888", "SeaweedFS filer server address")
|
||||
phase = flag.String("phase", "", "Phase to execute: generate, delete, status (for EC vacuum testing)")
|
||||
fileCount = flag.Int("files", 20, "Number of files to create")
|
||||
deleteRatio = flag.Float64("delete", 0.4, "Ratio of files to delete (0.0-1.0)")
|
||||
fileSizeKB = flag.Int("size", 100, "Size of each file in KB")
|
||||
)
|
||||
|
||||
type AssignResult struct {
|
||||
Fid string `json:"fid"`
|
||||
Url string `json:"url"`
|
||||
PublicUrl string `json:"publicUrl"`
|
||||
Count int `json:"count"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
// No longer needed - using filer-based operations
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
// Handle EC vacuum testing phases
|
||||
if *phase != "" {
|
||||
handleECVacuumPhase()
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("🧪 Creating fake data for vacuum task testing...")
|
||||
fmt.Printf("Master: %s\n", *master)
|
||||
fmt.Printf("Filer: %s\n", *filer)
|
||||
fmt.Printf("Files to create: %d\n", *fileCount)
|
||||
fmt.Printf("Delete ratio: %.1f%%\n", *deleteRatio*100)
|
||||
fmt.Printf("File size: %d KB\n", *fileSizeKB)
|
||||
@@ -46,11 +50,11 @@ func main() {
|
||||
|
||||
// Step 1: Create test files
|
||||
fmt.Println("📁 Step 1: Creating test files...")
|
||||
fids := createTestFiles()
|
||||
filePaths := createTestFiles()
|
||||
|
||||
// Step 2: Delete some files to create garbage
|
||||
fmt.Println("🗑️ Step 2: Deleting files to create garbage...")
|
||||
deleteFiles(fids)
|
||||
deleteFiles(filePaths)
|
||||
|
||||
// Step 3: Check volume status
|
||||
fmt.Println("📊 Step 3: Checking volume status...")
|
||||
@@ -62,45 +66,41 @@ func main() {
|
||||
}
|
||||
|
||||
func createTestFiles() []string {
|
||||
var fids []string
|
||||
var filePaths []string
|
||||
|
||||
for i := 0; i < *fileCount; i++ {
|
||||
// Generate random file content
|
||||
fileData := make([]byte, *fileSizeKB*1024)
|
||||
rand.Read(fileData)
|
||||
|
||||
// Get file ID assignment
|
||||
assign, err := assignFileId()
|
||||
// Create file path
|
||||
filePath := fmt.Sprintf("/vacuum_test/test_file_%d_%d.dat", time.Now().Unix(), i)
|
||||
|
||||
// Upload file to filer
|
||||
err := uploadFileToFiler(filePath, fileData)
|
||||
if err != nil {
|
||||
log.Printf("Failed to assign file ID for file %d: %v", i, err)
|
||||
log.Printf("Failed to upload file %d to filer: %v", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Upload file
|
||||
err = uploadFile(assign, fileData, fmt.Sprintf("test_file_%d.dat", i))
|
||||
if err != nil {
|
||||
log.Printf("Failed to upload file %d: %v", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
fids = append(fids, assign.Fid)
|
||||
filePaths = append(filePaths, filePath)
|
||||
|
||||
if (i+1)%5 == 0 {
|
||||
fmt.Printf(" Created %d/%d files...\n", i+1, *fileCount)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("✅ Created %d files successfully\n\n", len(fids))
|
||||
return fids
|
||||
fmt.Printf("✅ Created %d files successfully\n\n", len(filePaths))
|
||||
return filePaths
|
||||
}
|
||||
|
||||
func deleteFiles(fids []string) {
|
||||
deleteCount := int(float64(len(fids)) * *deleteRatio)
|
||||
func deleteFiles(filePaths []string) {
|
||||
deleteCount := int(float64(len(filePaths)) * *deleteRatio)
|
||||
|
||||
for i := 0; i < deleteCount; i++ {
|
||||
err := deleteFile(fids[i])
|
||||
err := deleteFileFromFiler(filePaths[i])
|
||||
if err != nil {
|
||||
log.Printf("Failed to delete file %s: %v", fids[i], err)
|
||||
log.Printf("Failed to delete file %s: %v", filePaths[i], err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -112,46 +112,23 @@ func deleteFiles(fids []string) {
|
||||
fmt.Printf("✅ Deleted %d files (%.1f%% of total)\n\n", deleteCount, *deleteRatio*100)
|
||||
}
|
||||
|
||||
func assignFileId() (*AssignResult, error) {
|
||||
resp, err := http.Get(fmt.Sprintf("http://%s/dir/assign", *master))
|
||||
// Filer-based functions for file operations
|
||||
|
||||
func uploadFileToFiler(filePath string, data []byte) error {
|
||||
url := fmt.Sprintf("http://%s%s", *filer, filePath)
|
||||
|
||||
req, err := http.NewRequest("PUT", url, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result AssignResult
|
||||
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if result.Error != "" {
|
||||
return nil, fmt.Errorf("assignment error: %s", result.Error)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func uploadFile(assign *AssignResult, data []byte, filename string) error {
|
||||
url := fmt.Sprintf("http://%s/%s", assign.Url, assign.Fid)
|
||||
|
||||
body := &bytes.Buffer{}
|
||||
body.Write(data)
|
||||
|
||||
req, err := http.NewRequest("POST", url, body)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to create request: %v", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
if filename != "" {
|
||||
req.Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
|
||||
}
|
||||
req.ContentLength = int64(len(data))
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to upload to filer: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
@@ -163,21 +140,28 @@ func uploadFile(assign *AssignResult, data []byte, filename string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteFile(fid string) error {
|
||||
url := fmt.Sprintf("http://%s/%s", *master, fid)
|
||||
func deleteFileFromFiler(filePath string) error {
|
||||
url := fmt.Sprintf("http://%s%s", *filer, filePath)
|
||||
|
||||
req, err := http.NewRequest("DELETE", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to create delete request: %v", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to delete from filer: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Accept both 204 (No Content) and 404 (Not Found) as success
|
||||
// 404 means file was already deleted
|
||||
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("delete failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -274,7 +258,320 @@ func printTestingInstructions() {
|
||||
fmt.Println(" Garbage ratios should decrease after vacuum operations")
|
||||
fmt.Println()
|
||||
|
||||
fmt.Printf("🚀 Quick test command:\n")
|
||||
fmt.Printf(" go run create_vacuum_test_data.go -files=0\n")
|
||||
fmt.Printf("🚀 Quick test commands:\n")
|
||||
fmt.Printf(" go run create_vacuum_test_data.go -files=0 # Check volume status\n")
|
||||
fmt.Printf(" go run create_vacuum_test_data.go -phase=status # Check EC volumes\n")
|
||||
fmt.Println()
|
||||
fmt.Println("💡 All operations now use the filer for realistic file management")
|
||||
}
|
||||
|
||||
// EC Vacuum Testing Functions
|
||||
|
||||
func handleECVacuumPhase() {
|
||||
fmt.Printf("🧪 EC Vacuum Test Data Script - Phase: %s\n", *phase)
|
||||
fmt.Printf("Master: %s\n", *master)
|
||||
fmt.Printf("Filer: %s\n", *filer)
|
||||
fmt.Println()
|
||||
|
||||
switch *phase {
|
||||
case "generate":
|
||||
generateECTestData()
|
||||
case "delete":
|
||||
deleteFromECVolumes()
|
||||
case "status":
|
||||
checkECVolumeStatus()
|
||||
default:
|
||||
fmt.Printf("❌ Unknown phase: %s\n", *phase)
|
||||
fmt.Println("Valid phases: generate, delete, status")
|
||||
}
|
||||
}
|
||||
|
||||
func generateECTestData() {
|
||||
fmt.Println("📁 Generating large files to trigger EC encoding...")
|
||||
fmt.Printf("Files to create: %d\n", *fileCount)
|
||||
fmt.Printf("File size: %d KB\n", *fileSizeKB)
|
||||
fmt.Printf("Filer: %s\n", *filer)
|
||||
fmt.Println()
|
||||
|
||||
var filePaths []string
|
||||
|
||||
for i := 0; i < *fileCount; i++ {
|
||||
// Generate random file content
|
||||
fileData := make([]byte, *fileSizeKB*1024)
|
||||
rand.Read(fileData)
|
||||
|
||||
// Create file path
|
||||
filePath := fmt.Sprintf("/ec_test/large_file_%d_%d.dat", time.Now().Unix(), i)
|
||||
|
||||
// Upload file to filer
|
||||
err := uploadFileToFiler(filePath, fileData)
|
||||
if err != nil {
|
||||
log.Printf("Failed to upload file %d to filer: %v", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
filePaths = append(filePaths, filePath)
|
||||
|
||||
if (i+1)%5 == 0 {
|
||||
fmt.Printf(" Created %d/%d files... (latest: %s)\n", i+1, *fileCount, filePath)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("✅ Created %d files successfully\n", len(filePaths))
|
||||
|
||||
// Store file paths for later deletion (using mounted working directory)
|
||||
err := storeFilePathsToFile(filePaths, "ec_test_files.json")
|
||||
if err != nil {
|
||||
fmt.Printf("⚠️ Warning: Failed to store file paths for deletion: %v\n", err)
|
||||
fmt.Println("💡 You can still test EC vacuum manually through the admin UI")
|
||||
} else {
|
||||
fmt.Printf("📝 Stored %d file paths for deletion phase\n", len(filePaths))
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("📊 Current volume status:")
|
||||
checkVolumeStatus()
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("⏳ Wait 2-3 minutes for EC encoding to complete...")
|
||||
fmt.Println("💡 EC encoding happens when volumes exceed 50MB")
|
||||
fmt.Println("💡 Run 'make ec-vacuum-status' to check EC volume creation")
|
||||
fmt.Println("💡 Then run 'make ec-vacuum-delete' to create garbage")
|
||||
}
|
||||
|
||||
func deleteFromECVolumes() {
|
||||
fmt.Printf("🗑️ Creating deletions on EC volumes (ratio: %.1f%%)\n", *deleteRatio*100)
|
||||
fmt.Printf("Filer: %s\n", *filer)
|
||||
fmt.Println()
|
||||
|
||||
// Load stored file paths from previous generation (using mounted working directory)
|
||||
filePaths, err := loadFilePathsFromFile("ec_test_files.json")
|
||||
if err != nil {
|
||||
fmt.Printf("❌ Failed to load stored file paths: %v\n", err)
|
||||
fmt.Println("💡 Run 'make ec-vacuum-generate' first to create files")
|
||||
return
|
||||
}
|
||||
|
||||
if len(filePaths) == 0 {
|
||||
fmt.Println("❌ No stored file paths found. Run generate phase first.")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("Found %d stored file paths from previous generation\n", len(filePaths))
|
||||
|
||||
deleteCount := int(float64(len(filePaths)) * *deleteRatio)
|
||||
fmt.Printf("Will delete %d files to create garbage\n", deleteCount)
|
||||
fmt.Println()
|
||||
|
||||
deletedCount := 0
|
||||
for i := 0; i < deleteCount && i < len(filePaths); i++ {
|
||||
err := deleteFileFromFiler(filePaths[i])
|
||||
if err != nil {
|
||||
log.Printf("Failed to delete file %s: %v", filePaths[i], err)
|
||||
} else {
|
||||
deletedCount++
|
||||
}
|
||||
|
||||
if (i+1)%5 == 0 {
|
||||
fmt.Printf(" Deleted %d/%d files...\n", i+1, deleteCount)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("✅ Successfully deleted %d files (%.1f%% of total)\n", deletedCount, *deleteRatio*100)
|
||||
fmt.Println()
|
||||
fmt.Println("📊 Updated status:")
|
||||
time.Sleep(5 * time.Second) // Wait for deletion to be processed
|
||||
checkECVolumeStatus()
|
||||
}
|
||||
|
||||
func checkECVolumeStatus() {
|
||||
fmt.Println("📊 EC Volume Status and Garbage Analysis")
|
||||
fmt.Println("========================================")
|
||||
|
||||
volumes := getVolumeStatusForDeletion()
|
||||
if len(volumes) == 0 {
|
||||
fmt.Println("❌ No volumes found")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("📈 Volume Analysis (potential EC candidates and EC volumes):")
|
||||
|
||||
regularECCandidates := 0
|
||||
ecVolumes := 0
|
||||
highGarbageCount := 0
|
||||
|
||||
for _, vol := range volumes {
|
||||
garbageRatio := 0.0
|
||||
if vol.Size > 0 {
|
||||
garbageRatio = float64(vol.DeletedByteCount) / float64(vol.Size) * 100
|
||||
}
|
||||
|
||||
status := "📁"
|
||||
volumeType := "Regular"
|
||||
|
||||
if vol.ReadOnly && vol.Size > 40*1024*1024 {
|
||||
status = "🔧"
|
||||
volumeType = "EC Volume"
|
||||
ecVolumes++
|
||||
if garbageRatio > 30 {
|
||||
status = "🧹"
|
||||
highGarbageCount++
|
||||
}
|
||||
} else if vol.Size > 40*1024*1024 {
|
||||
status = "📈"
|
||||
volumeType = "EC Candidate"
|
||||
regularECCandidates++
|
||||
}
|
||||
|
||||
fmt.Printf(" %s Volume %d (%s): %s, Files: %d/%d, Garbage: %.1f%%",
|
||||
status, vol.Id, volumeType, formatBytes(vol.Size), vol.FileCount, vol.DeleteCount, garbageRatio)
|
||||
|
||||
if volumeType == "EC Volume" && garbageRatio > 30 {
|
||||
fmt.Printf(" (Should trigger EC vacuum!)")
|
||||
}
|
||||
fmt.Printf("\n")
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("🎯 EC Vacuum Testing Summary:")
|
||||
fmt.Printf(" • Total volumes: %d\n", len(volumes))
|
||||
fmt.Printf(" • EC volumes (read-only >40MB): %d\n", ecVolumes)
|
||||
fmt.Printf(" • EC candidates (>40MB): %d\n", regularECCandidates)
|
||||
fmt.Printf(" • EC volumes with >30%% garbage: %d\n", highGarbageCount)
|
||||
|
||||
if highGarbageCount > 0 {
|
||||
fmt.Println()
|
||||
fmt.Println("✅ EC volumes with high garbage found!")
|
||||
fmt.Println("💡 Configure EC vacuum at: http://localhost:23646/maintenance/config/ec_vacuum")
|
||||
fmt.Println("💡 Monitor tasks at: http://localhost:23646/maintenance")
|
||||
} else if ecVolumes > 0 {
|
||||
fmt.Println()
|
||||
fmt.Println("ℹ️ EC volumes exist but garbage ratio is low")
|
||||
fmt.Println("💡 Run 'make ec-vacuum-delete' to create more garbage")
|
||||
} else if regularECCandidates > 0 {
|
||||
fmt.Println()
|
||||
fmt.Println("ℹ️ Large volumes found, waiting for EC encoding...")
|
||||
fmt.Println("💡 Wait a few more minutes for EC encoding to complete")
|
||||
} else {
|
||||
fmt.Println()
|
||||
fmt.Println("ℹ️ No large volumes found")
|
||||
fmt.Println("💡 Run 'make ec-vacuum-generate' to create large files for EC encoding")
|
||||
}
|
||||
}
|
||||
|
||||
type VolumeInfo struct {
|
||||
Id int `json:"Id"`
|
||||
Size uint64 `json:"Size"`
|
||||
FileCount int `json:"FileCount"`
|
||||
DeleteCount int `json:"DeleteCount"`
|
||||
DeletedByteCount uint64 `json:"DeletedByteCount"`
|
||||
ReadOnly bool `json:"ReadOnly"`
|
||||
Collection string `json:"Collection"`
|
||||
}
|
||||
|
||||
type VolumeStatus struct {
|
||||
Version string `json:"Version"`
|
||||
Volumes VolumeLayout `json:"Volumes"`
|
||||
}
|
||||
|
||||
type VolumeLayout struct {
|
||||
DataCenters map[string]map[string]map[string][]VolumeInfo `json:"DataCenters"`
|
||||
Free int `json:"Free"`
|
||||
Max int `json:"Max"`
|
||||
}
|
||||
|
||||
func getVolumeStatusForDeletion() []VolumeInfo {
|
||||
resp, err := http.Get(fmt.Sprintf("http://%s/vol/status", *master))
|
||||
if err != nil {
|
||||
log.Printf("Failed to get volume status: %v", err)
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var volumeStatus VolumeStatus
|
||||
err = json.NewDecoder(resp.Body).Decode(&volumeStatus)
|
||||
if err != nil {
|
||||
log.Printf("Failed to decode volume status: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Extract all volumes from the nested structure
|
||||
var allVolumes []VolumeInfo
|
||||
for dcName, dataCenter := range volumeStatus.Volumes.DataCenters {
|
||||
log.Printf("Processing data center: %s", dcName)
|
||||
for rackName, rack := range dataCenter {
|
||||
log.Printf("Processing rack: %s", rackName)
|
||||
for serverName, volumes := range rack {
|
||||
log.Printf("Found %d volumes on server %s", len(volumes), serverName)
|
||||
allVolumes = append(allVolumes, volumes...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allVolumes
|
||||
}
|
||||
|
||||
type StoredFilePaths struct {
|
||||
FilePaths []string `json:"file_paths"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
FileCount int `json:"file_count"`
|
||||
FileSize int `json:"file_size_kb"`
|
||||
}
|
||||
|
||||
func storeFilePathsToFile(filePaths []string, filename string) error {
|
||||
data := StoredFilePaths{
|
||||
FilePaths: filePaths,
|
||||
Timestamp: time.Now(),
|
||||
FileCount: len(filePaths),
|
||||
FileSize: *fileSizeKB,
|
||||
}
|
||||
|
||||
jsonData, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal file paths: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(filename, jsonData, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write file paths to file: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadFilePathsFromFile(filename string) ([]string, error) {
|
||||
// Check if file exists
|
||||
if _, err := os.Stat(filename); os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("file paths storage file does not exist: %s", filename)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file paths file: %v", err)
|
||||
}
|
||||
|
||||
var storedData StoredFilePaths
|
||||
err = json.Unmarshal(data, &storedData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal file paths: %v", err)
|
||||
}
|
||||
|
||||
// Check if data is recent (within last 24 hours)
|
||||
if time.Since(storedData.Timestamp) > 24*time.Hour {
|
||||
return nil, fmt.Errorf("stored file paths are too old (%v), please regenerate",
|
||||
time.Since(storedData.Timestamp))
|
||||
}
|
||||
|
||||
fmt.Printf("Loaded %d file paths from %v (File size: %dKB each)\n",
|
||||
len(storedData.FilePaths), storedData.Timestamp.Format("15:04:05"), storedData.FileSize)
|
||||
|
||||
return storedData.FilePaths, nil
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ services:
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "18080:18080"
|
||||
command: "volume -master=master:9333 -ip=volume1 -dir=/data -max=10"
|
||||
command: "-v=2 volume -mserver=master:9333 -ip=volume1 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -38,7 +38,7 @@ services:
|
||||
ports:
|
||||
- "8081:8080"
|
||||
- "18081:18080"
|
||||
command: "volume -master=master:9333 -ip=volume2 -dir=/data -max=10"
|
||||
command: "-v=2 volume -mserver=master:9333 -ip=volume2 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -51,7 +51,7 @@ services:
|
||||
ports:
|
||||
- "8082:8080"
|
||||
- "18082:18080"
|
||||
command: "volume -master=master:9333 -ip=volume3 -dir=/data -max=10"
|
||||
command: "-v=2 volume -mserver=master:9333 -ip=volume3 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -64,7 +64,7 @@ services:
|
||||
ports:
|
||||
- "8083:8080"
|
||||
- "18083:18080"
|
||||
command: "volume -master=master:9333 -ip=volume4 -dir=/data -max=10"
|
||||
command: "-v=2 volume -mserver=master:9333 -ip=volume4 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -77,7 +77,7 @@ services:
|
||||
ports:
|
||||
- "8084:8080"
|
||||
- "18084:18080"
|
||||
command: "volume -master=master:9333 -ip=volume5 -dir=/data -max=10"
|
||||
command: "-v=2 volume -mserver=master:9333 -ip=volume5 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -90,7 +90,7 @@ services:
|
||||
ports:
|
||||
- "8085:8080"
|
||||
- "18085:18080"
|
||||
command: "volume -master=master:9333 -ip=volume6 -dir=/data -max=10"
|
||||
command: "-v=2 volume -mserver=master:9333 -ip=volume6 -dir=/data -max=10"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
@@ -98,16 +98,42 @@ services:
|
||||
networks:
|
||||
- seaweed_net
|
||||
|
||||
filer:
|
||||
filer1:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- "8888:8888"
|
||||
- "18888:18888"
|
||||
command: "filer -master=master:9333 -ip=filer"
|
||||
command: "-v=2 filer -master=master:9333 -ip=filer1"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
- ./data/filer:/data
|
||||
- ./data/filer1:/data
|
||||
networks:
|
||||
- seaweed_net
|
||||
|
||||
filer2:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- "8889:8888"
|
||||
- "18889:18888"
|
||||
command: "-v=2 filer -master=master:9333 -ip=filer2"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
- ./data/filer2:/data
|
||||
networks:
|
||||
- seaweed_net
|
||||
|
||||
filer3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- "8890:8888"
|
||||
- "18890:18888"
|
||||
command: "-v=2 filer -master=master:9333 -ip=filer3"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
- ./data/filer3:/data
|
||||
networks:
|
||||
- seaweed_net
|
||||
|
||||
@@ -119,7 +145,7 @@ services:
|
||||
command: "-v=2 admin -port=23646 -masters=master:9333 -dataDir=/data"
|
||||
depends_on:
|
||||
- master
|
||||
- filer
|
||||
- filer1
|
||||
volumes:
|
||||
- ./data/admin:/data
|
||||
networks:
|
||||
@@ -127,7 +153,7 @@ services:
|
||||
|
||||
worker1:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,ec_vacuum -maxConcurrent=2"
|
||||
depends_on:
|
||||
- admin
|
||||
volumes:
|
||||
@@ -139,7 +165,7 @@ services:
|
||||
|
||||
worker2:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,ec_vacuum -maxConcurrent=2"
|
||||
depends_on:
|
||||
- admin
|
||||
volumes:
|
||||
@@ -151,7 +177,7 @@ services:
|
||||
|
||||
worker3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,ec_vacuum -maxConcurrent=2"
|
||||
depends_on:
|
||||
- admin
|
||||
volumes:
|
||||
@@ -180,7 +206,7 @@ services:
|
||||
"
|
||||
depends_on:
|
||||
- master
|
||||
- filer
|
||||
- filer1
|
||||
- admin
|
||||
networks:
|
||||
- seaweed_net
|
||||
@@ -210,7 +236,7 @@ services:
|
||||
depends_on:
|
||||
- master
|
||||
- admin
|
||||
- filer
|
||||
- filer1
|
||||
networks:
|
||||
- seaweed_net
|
||||
|
||||
@@ -229,7 +255,7 @@ services:
|
||||
depends_on:
|
||||
- master
|
||||
- admin
|
||||
- filer
|
||||
- filer1
|
||||
volumes:
|
||||
- .:/testing
|
||||
working_dir: /testing
|
||||
|
||||
@@ -0,0 +1,341 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestFileGeneration tests generating 600 files of 100KB each targeting volume 1 with hardcoded cookie
|
||||
func TestFileGeneration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping file generation test in short mode")
|
||||
}
|
||||
|
||||
// Set up test cluster
|
||||
cluster, cleanup := setupTestCluster(t)
|
||||
defer cleanup()
|
||||
|
||||
// Wait for cluster to be ready
|
||||
require.NoError(t, waitForClusterReady())
|
||||
t.Logf("Test cluster ready with master at %s", cluster.masterAddress)
|
||||
|
||||
// Generate 600 files of 100KB each targeting volume 1
|
||||
const targetVolumeId = needle.VolumeId(1)
|
||||
const fileCount = 600
|
||||
const fileSize = 100 * 1024 // 100KB files
|
||||
|
||||
fileIds := generateFilesToVolume1(t, fileCount, fileSize)
|
||||
t.Logf("Generated %d files of %dKB each targeting volume %d", len(fileIds), fileSize/1024, targetVolumeId)
|
||||
t.Logf("📝 Sample file IDs created: %v", fileIds[:5]) // Show first 5 file IDs
|
||||
|
||||
// Summary
|
||||
t.Logf("📊 File Generation Summary:")
|
||||
t.Logf(" • Volume ID: %d", targetVolumeId)
|
||||
t.Logf(" • Total files created: %d", len(fileIds))
|
||||
t.Logf(" • File size: %dKB each", fileSize/1024)
|
||||
t.Logf(" • Hardcoded cookie: 0x12345678")
|
||||
}
|
||||
|
||||
// TestFileDeletion tests deleting exactly 300 files from volume 1
|
||||
func TestFileDeletion(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping file deletion test in short mode")
|
||||
}
|
||||
|
||||
// Set up test cluster
|
||||
cluster, cleanup := setupTestCluster(t)
|
||||
defer cleanup()
|
||||
|
||||
// Wait for cluster to be ready
|
||||
require.NoError(t, waitForClusterReady())
|
||||
t.Logf("Test cluster ready with master at %s", cluster.masterAddress)
|
||||
|
||||
// First generate some files to delete
|
||||
const targetVolumeId = needle.VolumeId(1)
|
||||
const fileCount = 600
|
||||
const fileSize = 100 * 1024 // 100KB files
|
||||
|
||||
t.Logf("Pre-generating files for deletion test...")
|
||||
fileIds := generateFilesToVolume1(t, fileCount, fileSize)
|
||||
t.Logf("Pre-generated %d files for deletion test", len(fileIds))
|
||||
|
||||
// Delete exactly 300 files from the volume
|
||||
const deleteCount = 300
|
||||
deletedFileIds := deleteSpecificFilesFromVolume(t, fileIds, deleteCount)
|
||||
t.Logf("Deleted exactly %d files from volume %d", len(deletedFileIds), targetVolumeId)
|
||||
t.Logf("🗑️ Sample deleted file IDs: %v", deletedFileIds[:5]) // Show first 5 deleted file IDs
|
||||
|
||||
// Summary
|
||||
t.Logf("📊 File Deletion Summary:")
|
||||
t.Logf(" • Volume ID: %d", targetVolumeId)
|
||||
t.Logf(" • Files available for deletion: %d", len(fileIds))
|
||||
t.Logf(" • Files deleted: %d", len(deletedFileIds))
|
||||
t.Logf(" • Files remaining: %d", len(fileIds)-len(deletedFileIds))
|
||||
t.Logf(" • Deletion success rate: %.1f%%", float64(len(deletedFileIds))/float64(deleteCount)*100)
|
||||
}
|
||||
|
||||
// generateFilesToVolume1 creates 600 files of 100KB each targeting volume 1 with hardcoded cookie
|
||||
func generateFilesToVolume1(t *testing.T, fileCount int, fileSize int) []string {
|
||||
const targetVolumeId = needle.VolumeId(1)
|
||||
const hardcodedCookie = uint32(0x12345678) // Hardcoded cookie for volume 1 files
|
||||
|
||||
var fileIds []string
|
||||
t.Logf("Starting generation of %d files of %dKB each targeting volume %d", fileCount, fileSize/1024, targetVolumeId)
|
||||
|
||||
for i := 0; i < fileCount; i++ {
|
||||
// Generate file content
|
||||
fileData := make([]byte, fileSize)
|
||||
rand.Read(fileData)
|
||||
|
||||
// Generate file ID targeting volume 1 with hardcoded cookie
|
||||
// Use high needle key values to avoid collisions with assigned IDs
|
||||
needleKey := uint64(i) + 0x2000000 // Start from a high offset for volume 1
|
||||
generatedFid := needle.NewFileId(targetVolumeId, needleKey, hardcodedCookie)
|
||||
|
||||
// Upload directly to volume 1
|
||||
err := uploadFileToVolumeDirectly(t, generatedFid, fileData)
|
||||
require.NoError(t, err)
|
||||
|
||||
fileIds = append(fileIds, generatedFid.String())
|
||||
|
||||
// Log progress for first few files and every 50th file
|
||||
if i < 5 || (i+1)%50 == 0 {
|
||||
t.Logf("✅ Generated file %d/%d targeting volume %d: %s", i+1, fileCount, targetVolumeId, generatedFid.String())
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("✅ Successfully generated %d files targeting volume %d with hardcoded cookie 0x%08x", len(fileIds), targetVolumeId, hardcodedCookie)
|
||||
return fileIds
|
||||
}
|
||||
|
||||
// uploadFileToVolume uploads file data to an assigned volume server
|
||||
func uploadFileToVolume(serverUrl, fid string, data []byte) error {
|
||||
uploadUrl := fmt.Sprintf("http://%s/%s", serverUrl, fid)
|
||||
|
||||
req, err := http.NewRequest("PUT", uploadUrl, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFileToVolumeDirectly uploads a file using a generated file ID
|
||||
func uploadFileToVolumeDirectly(t *testing.T, fid *needle.FileId, data []byte) error {
|
||||
// Find the volume server hosting this volume
|
||||
locations, found := findVolumeLocations(fid.VolumeId)
|
||||
if !found {
|
||||
return fmt.Errorf("volume %d not found", fid.VolumeId)
|
||||
}
|
||||
|
||||
// Upload to the first available location
|
||||
serverUrl := locations[0]
|
||||
uploadUrl := fmt.Sprintf("http://%s/%s", serverUrl, fid.String())
|
||||
|
||||
req, err := http.NewRequest("PUT", uploadUrl, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("direct upload failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// findVolumeLocations finds the server locations for a given volume using HTTP lookup
|
||||
func findVolumeLocations(volumeId needle.VolumeId) ([]string, bool) {
|
||||
// Query master for volume locations using HTTP API
|
||||
resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:9333/dir/lookup?volumeId=%d", volumeId))
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Parse JSON response
|
||||
type LookupResult struct {
|
||||
VolumeId string `json:"volumeId"`
|
||||
Locations []struct {
|
||||
Url string `json:"url"`
|
||||
PublicUrl string `json:"publicUrl"`
|
||||
} `json:"locations"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
var result LookupResult
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
// Fallback to default locations for testing
|
||||
return []string{"127.0.0.1:8080"}, true
|
||||
}
|
||||
|
||||
if result.Error != "" {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var serverUrls []string
|
||||
for _, location := range result.Locations {
|
||||
// Convert Docker container hostnames to localhost with mapped ports
|
||||
url := convertDockerHostnameToLocalhost(location.Url)
|
||||
serverUrls = append(serverUrls, url)
|
||||
}
|
||||
|
||||
if len(serverUrls) == 0 {
|
||||
// Fallback to default for testing
|
||||
return []string{"127.0.0.1:8080"}, true
|
||||
}
|
||||
|
||||
return serverUrls, true
|
||||
}
|
||||
|
||||
// convertDockerHostnameToLocalhost converts Docker container hostnames to localhost with mapped ports
|
||||
func convertDockerHostnameToLocalhost(dockerUrl string) string {
|
||||
// Map Docker container hostnames to localhost ports
|
||||
hostPortMap := map[string]string{
|
||||
"volume1:8080": "127.0.0.1:8080",
|
||||
"volume2:8080": "127.0.0.1:8081",
|
||||
"volume3:8080": "127.0.0.1:8082",
|
||||
"volume4:8080": "127.0.0.1:8083",
|
||||
"volume5:8080": "127.0.0.1:8084",
|
||||
"volume6:8080": "127.0.0.1:8085",
|
||||
}
|
||||
|
||||
if localhost, exists := hostPortMap[dockerUrl]; exists {
|
||||
return localhost
|
||||
}
|
||||
|
||||
// If not in map, return as-is (might be already localhost)
|
||||
return dockerUrl
|
||||
}
|
||||
|
||||
// deleteSpecificFilesFromVolume deletes exactly the specified number of files from the volume
|
||||
func deleteSpecificFilesFromVolume(t *testing.T, fileIds []string, deleteCount int) []string {
|
||||
var deletedFileIds []string
|
||||
successfulDeletions := 0
|
||||
|
||||
if deleteCount > len(fileIds) {
|
||||
deleteCount = len(fileIds)
|
||||
}
|
||||
|
||||
t.Logf("🗑️ Starting deletion of exactly %d files out of %d total files", deleteCount, len(fileIds))
|
||||
|
||||
for i := 0; i < deleteCount; i++ {
|
||||
fileId := fileIds[i]
|
||||
|
||||
// Parse file ID to get volume server location
|
||||
fid, err := needle.ParseFileIdFromString(fileId)
|
||||
if err != nil {
|
||||
t.Logf("Failed to parse file ID %s: %v", fileId, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Find volume server hosting this file
|
||||
locations, found := findVolumeLocations(fid.VolumeId)
|
||||
if !found {
|
||||
t.Logf("Volume locations not found for file %s", fileId)
|
||||
continue
|
||||
}
|
||||
|
||||
// Delete file from volume server
|
||||
deleteUrl := fmt.Sprintf("http://%s/%s", locations[0], fileId)
|
||||
req, err := http.NewRequest("DELETE", deleteUrl, nil)
|
||||
if err != nil {
|
||||
t.Logf("Failed to create delete request for file %s: %v", fileId, err)
|
||||
continue
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Logf("Failed to delete file %s: %v", fileId, err)
|
||||
continue
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusOK {
|
||||
successfulDeletions++
|
||||
deletedFileIds = append(deletedFileIds, fileId)
|
||||
// Log progress for first few files and every 25th deletion
|
||||
if i < 5 || (i+1)%25 == 0 {
|
||||
t.Logf("🗑️ Deleted file %d/%d: %s (status: %d)", i+1, deleteCount, fileId, resp.StatusCode)
|
||||
}
|
||||
} else {
|
||||
t.Logf("❌ Delete failed for file %s with status %d", fileId, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("✅ Deletion summary: %d files deleted successfully out of %d attempted", successfulDeletions, deleteCount)
|
||||
return deletedFileIds
|
||||
}
|
||||
|
||||
// Helper functions for test setup
|
||||
|
||||
func setupTestCluster(t *testing.T) (*TestCluster, func()) {
|
||||
// Create test cluster similar to existing integration tests
|
||||
// This is a simplified version - in practice would start actual servers
|
||||
cluster := &TestCluster{
|
||||
masterAddress: "127.0.0.1:9333",
|
||||
volumeServers: []string{
|
||||
"127.0.0.1:8080",
|
||||
"127.0.0.1:8081",
|
||||
"127.0.0.1:8082",
|
||||
"127.0.0.1:8083",
|
||||
"127.0.0.1:8084",
|
||||
"127.0.0.1:8085",
|
||||
},
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
// Cleanup cluster resources
|
||||
t.Logf("Cleaning up test cluster")
|
||||
}
|
||||
|
||||
return cluster, cleanup
|
||||
}
|
||||
|
||||
func waitForClusterReady() error {
|
||||
// Wait for test cluster to be ready
|
||||
// In practice, this would ping the servers and wait for them to respond
|
||||
time.Sleep(2 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
type TestCluster struct {
|
||||
masterAddress string
|
||||
volumeServers []string
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
# Filer Benchmark Tool
|
||||
|
||||
A simple Go program to benchmark SeaweedFS filer performance and detect race conditions with concurrent file operations.
|
||||
|
||||
## Overview
|
||||
|
||||
This tool creates 300 (configurable) goroutines that concurrently:
|
||||
1. Create empty files on the filer
|
||||
2. Add multiple chunks to each file (with fake file IDs)
|
||||
3. Verify the file was created successfully
|
||||
|
||||
This simulates the race condition scenario from [Issue #7062](https://github.com/seaweedfs/seaweedfs/issues/7062) where concurrent operations can lead to metadata inconsistencies.
|
||||
|
||||
## Usage
|
||||
|
||||
### Build and Run Directly
|
||||
```bash
|
||||
# Build the tool
|
||||
go build -o bin/filer_benchmark ./cmd/filer_benchmark/
|
||||
|
||||
# Basic usage (single filer)
|
||||
./bin/filer_benchmark -filers=localhost:8888
|
||||
|
||||
# Test with multiple filers
|
||||
./bin/filer_benchmark -filers=localhost:8888,localhost:8889,localhost:8890
|
||||
|
||||
# High concurrency race condition test
|
||||
./bin/filer_benchmark -goroutines=500 -loops=200 -verbose
|
||||
```
|
||||
|
||||
### Using Helper Scripts
|
||||
```bash
|
||||
# Use the wrapper script with predefined configurations
|
||||
./scripts/run_filer_benchmark.sh
|
||||
|
||||
# Run example test suite
|
||||
./examples/run_filer_race_test.sh
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `-filers` | `localhost:8888` | Comma-separated list of filer addresses |
|
||||
| `-goroutines` | `300` | Number of concurrent goroutines |
|
||||
| `-loops` | `100` | Number of operations per goroutine |
|
||||
| `-chunkSize` | `1048576` | Chunk size in bytes (1MB) |
|
||||
| `-chunksPerFile` | `5` | Number of chunks per file |
|
||||
| `-testDir` | `/benchmark` | Test directory on filer |
|
||||
| `-verbose` | `false` | Enable verbose error logging |
|
||||
|
||||
## Race Condition Detection
|
||||
|
||||
The tool detects race conditions by monitoring for these error patterns:
|
||||
- `leveldb: closed` - Metadata cache closed during operation
|
||||
- `transport is closing` - gRPC connection closed during operation
|
||||
- `connection refused` - Network connectivity issues
|
||||
- `not found after creation` - File disappeared after being created
|
||||
|
||||
## Example Output
|
||||
|
||||
```
|
||||
============================================================
|
||||
FILER BENCHMARK RESULTS
|
||||
============================================================
|
||||
Configuration:
|
||||
Filers: localhost:8888,localhost:8889,localhost:8890
|
||||
Goroutines: 300
|
||||
Loops per goroutine: 100
|
||||
Chunks per file: 5
|
||||
Chunk size: 1048576 bytes
|
||||
|
||||
Results:
|
||||
Total operations attempted: 30000
|
||||
Files successfully created: 29850
|
||||
Total chunks added: 149250
|
||||
Errors: 150
|
||||
Race condition errors: 23
|
||||
Success rate: 99.50%
|
||||
|
||||
Performance:
|
||||
Total duration: 45.2s
|
||||
Operations/second: 663.72
|
||||
Files/second: 660.18
|
||||
Chunks/second: 3300.88
|
||||
|
||||
Race Condition Analysis:
|
||||
Race condition rate: 0.0767%
|
||||
Race conditions detected: 23
|
||||
🟡 MODERATE race condition rate
|
||||
Overall error rate: 0.50%
|
||||
============================================================
|
||||
```
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### 1. Basic Functionality Test
|
||||
```bash
|
||||
./bin/filer_benchmark -goroutines=20 -loops=10
|
||||
```
|
||||
Low concurrency test to verify basic functionality.
|
||||
|
||||
### 2. Race Condition Reproduction
|
||||
```bash
|
||||
./bin/filer_benchmark -goroutines=500 -loops=100 -verbose
|
||||
```
|
||||
High concurrency test designed to trigger race conditions.
|
||||
|
||||
### 3. Multi-Filer Load Test
|
||||
```bash
|
||||
./bin/filer_benchmark -filers=filer1:8888,filer2:8888,filer3:8888 -goroutines=300
|
||||
```
|
||||
Distribute load across multiple filers.
|
||||
|
||||
### 4. Small Files Benchmark
|
||||
```bash
|
||||
./bin/filer_benchmark -chunkSize=4096 -chunksPerFile=1 -goroutines=1000
|
||||
```
|
||||
Test with many small files to stress metadata operations.
|
||||
|
||||
## How It Simulates Race Conditions
|
||||
|
||||
1. **Concurrent Operations**: Multiple goroutines perform file operations simultaneously
|
||||
2. **Random Timing**: Small random delays create timing variations
|
||||
3. **Fake Chunks**: Uses file IDs without actual volume server data to focus on metadata operations
|
||||
4. **Verification Step**: Attempts to read files immediately after creation to catch race conditions
|
||||
5. **Multiple Filers**: Distributes load randomly across multiple filer instances
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- SeaweedFS master server running
|
||||
- SeaweedFS filer server(s) running
|
||||
- Go 1.19+ for building
|
||||
- Network connectivity to filer endpoints
|
||||
|
||||
## Integration with Issue #7062
|
||||
|
||||
This tool reproduces the core problem from the original issue:
|
||||
- **Concurrent file operations** (simulated by goroutines)
|
||||
- **Metadata race conditions** (detected through error patterns)
|
||||
- **Transport disconnections** (monitored in error analysis)
|
||||
- **File inconsistencies** (caught by verification steps)
|
||||
|
||||
The key difference is this tool focuses on the filer metadata layer rather than the full CSI driver + mount stack, making it easier to isolate and debug the race condition.
|
||||
|
||||
## Debugging Findings
|
||||
|
||||
### Multi-Filer vs Single-Filer Connection Issue
|
||||
|
||||
**Problem**: When using multiple filers with independent stores (non-shared backend), the benchmark may fail with errors like:
|
||||
- `update entry with chunks failed: rpc error: code = Unknown desc = not found /benchmark/file_X: filer: no entry is found in filer store`
|
||||
- `CreateEntry /benchmark/file_X: /benchmark should be a directory`
|
||||
|
||||
**Root Cause**: The issue is NOT missing metadata events, but rather the benchmark's round-robin load balancing across filers:
|
||||
|
||||
1. **File Creation**: Benchmark creates `file_X` on `filer1`
|
||||
2. **Chunk Updates**: Benchmark tries to update `file_X` on `filer2` or `filer3`
|
||||
3. **Error**: `filer2`/`filer3` don't have `file_X` in their local store yet (metadata sync delay)
|
||||
|
||||
**Verification**: Running with single filer connection (`-filers localhost:18888`) while 3 filers are running shows **NO missed events**, confirming metadata synchronization works correctly.
|
||||
|
||||
**Solutions**:
|
||||
- Ensure `/benchmark` directory exists on ALL filers before starting
|
||||
- Use file affinity (same filer for create/update operations)
|
||||
- Add retry logic for cross-filer operations
|
||||
- Add small delays to allow metadata sync between operations
|
||||
@@ -0,0 +1,363 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
var (
|
||||
filers = flag.String("filers", "localhost:8888", "comma-separated list of filer addresses")
|
||||
workers = flag.Int("workers", 300, "number of concurrent workers")
|
||||
threadsPerWorker = flag.Int("threadsPerWorker", 4, "number of threads per worker")
|
||||
concurrentFiles = flag.Int("concurrentFiles", 16, "number of files open concurrently per worker")
|
||||
filesPerWorker = flag.Int("filesPerWorker", 4096, "total number of files each worker creates")
|
||||
chunkSize = flag.Int64("chunkSize", 1024*1024, "chunk size in bytes")
|
||||
chunksPerFile = flag.Int("chunksPerFile", 5, "number of chunks per file")
|
||||
testDir = flag.String("testDir", "/benchmark", "test directory on filer")
|
||||
verbose = flag.Bool("verbose", false, "verbose logging")
|
||||
)
|
||||
|
||||
type BenchmarkStats struct {
|
||||
filesCreated int64
|
||||
chunksAdded int64
|
||||
errors int64
|
||||
raceConditions int64
|
||||
totalDuration time.Duration
|
||||
}
|
||||
|
||||
type FilerClient struct {
|
||||
address string
|
||||
conn *grpc.ClientConn
|
||||
client filer_pb.SeaweedFilerClient
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
// Configure logging based on verbose flag
|
||||
if !*verbose {
|
||||
log.SetFlags(log.LstdFlags) // Minimal logging
|
||||
}
|
||||
|
||||
filerAddresses := util.StringSplit(*filers, ",")
|
||||
if len(filerAddresses) == 0 {
|
||||
log.Fatal("No filer addresses provided")
|
||||
}
|
||||
|
||||
log.Printf("Starting filer benchmark: %d workers, %d threads each, %d concurrent files, %d files per worker, %d filers",
|
||||
*workers, *threadsPerWorker, *concurrentFiles, *filesPerWorker, len(filerAddresses))
|
||||
|
||||
// Create filer clients
|
||||
clients, err := createFilerClients(filerAddresses)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create filer clients: %v", err)
|
||||
}
|
||||
defer closeFilerClients(clients)
|
||||
|
||||
// Ensure test directory exists
|
||||
if err := ensureDirectory(clients[0], *testDir); err != nil {
|
||||
log.Fatalf("Failed to create test directory: %v", err)
|
||||
}
|
||||
|
||||
// Run benchmark
|
||||
stats := runBenchmark(clients)
|
||||
|
||||
// Print results
|
||||
printResults(stats)
|
||||
}
|
||||
|
||||
func createFilerClients(addresses []string) ([]*FilerClient, error) {
|
||||
var clients []*FilerClient
|
||||
|
||||
util.LoadSecurityConfiguration()
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
|
||||
|
||||
for _, addr := range addresses {
|
||||
conn, err := pb.GrpcDial(context.Background(), addr, true, grpcDialOption)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to %s: %v", addr, err)
|
||||
}
|
||||
|
||||
client := &FilerClient{
|
||||
address: addr,
|
||||
conn: conn,
|
||||
client: filer_pb.NewSeaweedFilerClient(conn),
|
||||
}
|
||||
clients = append(clients, client)
|
||||
}
|
||||
|
||||
return clients, nil
|
||||
}
|
||||
|
||||
func closeFilerClients(clients []*FilerClient) {
|
||||
for _, client := range clients {
|
||||
client.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func ensureDirectory(client *FilerClient, dir string) error {
|
||||
_, err := client.client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
|
||||
Directory: "/",
|
||||
Entry: &filer_pb.Entry{
|
||||
Name: dir[1:], // Remove leading slash
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
Mtime: time.Now().Unix(),
|
||||
Crtime: time.Now().Unix(),
|
||||
FileMode: 0755,
|
||||
},
|
||||
},
|
||||
OExcl: false,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func runBenchmark(clients []*FilerClient) *BenchmarkStats {
|
||||
stats := &BenchmarkStats{}
|
||||
var wg sync.WaitGroup
|
||||
startTime := time.Now()
|
||||
|
||||
// Start workers
|
||||
for i := 0; i < *workers; i++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
runWorker(workerID, clients, stats)
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Wait for completion
|
||||
wg.Wait()
|
||||
stats.totalDuration = time.Since(startTime)
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
func runWorker(workerID int, clients []*FilerClient, stats *BenchmarkStats) {
|
||||
// Create work queue with concurrency limit
|
||||
workQueue := make(chan int, *concurrentFiles)
|
||||
var workerWg sync.WaitGroup
|
||||
|
||||
// Start threads for this worker
|
||||
for threadID := 0; threadID < *threadsPerWorker; threadID++ {
|
||||
workerWg.Add(1)
|
||||
go func(tID int) {
|
||||
defer workerWg.Done()
|
||||
runWorkerThread(workerID, tID, clients, stats, workQueue)
|
||||
}(threadID)
|
||||
}
|
||||
|
||||
// Queue up all the file creation tasks
|
||||
go func() {
|
||||
defer close(workQueue)
|
||||
for fileID := 0; fileID < *filesPerWorker; fileID++ {
|
||||
workQueue <- fileID
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for all threads in this worker to complete
|
||||
workerWg.Wait()
|
||||
}
|
||||
|
||||
func runWorkerThread(workerID, threadID int, clients []*FilerClient, stats *BenchmarkStats, workQueue <-chan int) {
|
||||
for fileID := range workQueue {
|
||||
// Select random filer client
|
||||
client := clients[rand.Intn(len(clients))]
|
||||
|
||||
// Create unique filename
|
||||
filename := fmt.Sprintf("file_%d_%d_%d_%d", workerID, threadID, fileID, time.Now().UnixNano())
|
||||
|
||||
if err := createFileWithChunks(client, filename, stats); err != nil {
|
||||
atomic.AddInt64(&stats.errors, 1)
|
||||
if isRaceConditionError(err) {
|
||||
atomic.AddInt64(&stats.raceConditions, 1)
|
||||
}
|
||||
if *verbose {
|
||||
log.Printf("Worker %d Thread %d error: %v", workerID, threadID, err)
|
||||
}
|
||||
} else {
|
||||
atomic.AddInt64(&stats.filesCreated, 1)
|
||||
}
|
||||
|
||||
// Small random delay to create timing variations
|
||||
if rand.Intn(10) == 0 {
|
||||
time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createFileWithChunks(client *FilerClient, filename string, stats *BenchmarkStats) error {
|
||||
ctx := context.Background()
|
||||
|
||||
// Step 1: Create empty file
|
||||
entry := &filer_pb.Entry{
|
||||
Name: filename,
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
Mtime: time.Now().Unix(),
|
||||
Crtime: time.Now().Unix(),
|
||||
FileMode: 0644,
|
||||
FileSize: 0,
|
||||
},
|
||||
Chunks: []*filer_pb.FileChunk{},
|
||||
}
|
||||
|
||||
_, err := client.client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
|
||||
Directory: *testDir,
|
||||
Entry: entry,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create entry failed: %v", err)
|
||||
}
|
||||
|
||||
// Step 2: Add chunks to the file
|
||||
var chunks []*filer_pb.FileChunk
|
||||
var offset int64 = 0
|
||||
|
||||
for i := 0; i < *chunksPerFile; i++ {
|
||||
chunk := &filer_pb.FileChunk{
|
||||
FileId: generateFakeFileId(),
|
||||
Offset: offset,
|
||||
Size: uint64(*chunkSize),
|
||||
ModifiedTsNs: time.Now().UnixNano(),
|
||||
ETag: generateETag(),
|
||||
}
|
||||
chunks = append(chunks, chunk)
|
||||
offset += *chunkSize
|
||||
atomic.AddInt64(&stats.chunksAdded, 1)
|
||||
}
|
||||
|
||||
// Update file with chunks
|
||||
entry.Chunks = chunks
|
||||
entry.Attributes.FileSize = uint64(offset)
|
||||
|
||||
_, err = client.client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
|
||||
Directory: *testDir,
|
||||
Entry: entry,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("update entry with chunks failed: %v", err)
|
||||
}
|
||||
|
||||
// Step 3: Verify file was created properly (this may catch race conditions)
|
||||
_, err = client.client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: *testDir,
|
||||
Name: filename,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup after creation failed (race condition?): %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func generateFakeFileId() string {
|
||||
// Generate fake file ID that looks real but doesn't exist on volume servers
|
||||
volumeId := rand.Intn(100) + 1
|
||||
fileKey := rand.Int63()
|
||||
cookie := rand.Uint32()
|
||||
return fmt.Sprintf("%d,%x%08x", volumeId, fileKey, cookie)
|
||||
}
|
||||
|
||||
func generateETag() string {
|
||||
// Generate fake ETag
|
||||
return fmt.Sprintf("%x", rand.Int63())
|
||||
}
|
||||
|
||||
func isRaceConditionError(err error) bool {
|
||||
errStr := err.Error()
|
||||
return strings.Contains(errStr, "leveldb: closed") ||
|
||||
strings.Contains(errStr, "transport is closing") ||
|
||||
strings.Contains(errStr, "connection refused") ||
|
||||
strings.Contains(errStr, "not found") && strings.Contains(errStr, "after creation")
|
||||
}
|
||||
|
||||
func printResults(stats *BenchmarkStats) {
|
||||
fmt.Println("\n" + strings.Repeat("=", 60))
|
||||
fmt.Println("FILER BENCHMARK RESULTS")
|
||||
fmt.Println(strings.Repeat("=", 60))
|
||||
|
||||
totalOps := int64(*workers) * int64(*filesPerWorker)
|
||||
successRate := float64(stats.filesCreated) / float64(totalOps) * 100
|
||||
|
||||
fmt.Printf("Configuration:\n")
|
||||
fmt.Printf(" Filers: %s\n", *filers)
|
||||
fmt.Printf(" Workers: %d\n", *workers)
|
||||
fmt.Printf(" Threads per worker: %d\n", *threadsPerWorker)
|
||||
fmt.Printf(" Concurrent files per worker: %d\n", *concurrentFiles)
|
||||
fmt.Printf(" Files per worker: %d\n", *filesPerWorker)
|
||||
fmt.Printf(" Total threads: %d\n", *workers**threadsPerWorker)
|
||||
fmt.Printf(" Chunks per file: %d\n", *chunksPerFile)
|
||||
fmt.Printf(" Chunk size: %d bytes\n", *chunkSize)
|
||||
fmt.Printf("\n")
|
||||
|
||||
fmt.Printf("Results:\n")
|
||||
fmt.Printf(" Total operations attempted: %d\n", totalOps)
|
||||
fmt.Printf(" Files successfully created: %d\n", stats.filesCreated)
|
||||
fmt.Printf(" Total chunks added: %d\n", stats.chunksAdded)
|
||||
fmt.Printf(" Errors: %d\n", stats.errors)
|
||||
fmt.Printf(" Race condition errors: %d\n", stats.raceConditions)
|
||||
fmt.Printf(" Success rate: %.2f%%\n", successRate)
|
||||
fmt.Printf("\n")
|
||||
|
||||
fmt.Printf("Performance:\n")
|
||||
fmt.Printf(" Total duration: %v\n", stats.totalDuration)
|
||||
fmt.Printf(" Operations/second: %.2f\n", float64(totalOps)/stats.totalDuration.Seconds())
|
||||
fmt.Printf(" Files/second: %.2f\n", float64(stats.filesCreated)/stats.totalDuration.Seconds())
|
||||
fmt.Printf(" Chunks/second: %.2f\n", float64(stats.chunksAdded)/stats.totalDuration.Seconds())
|
||||
fmt.Printf("\n")
|
||||
|
||||
// Race condition analysis
|
||||
fmt.Printf("Race Condition Analysis:\n")
|
||||
if stats.raceConditions > 0 {
|
||||
raceRate := float64(stats.raceConditions) / float64(totalOps) * 100
|
||||
fmt.Printf(" Race condition rate: %.4f%%\n", raceRate)
|
||||
fmt.Printf(" Race conditions detected: %d\n", stats.raceConditions)
|
||||
|
||||
if raceRate > 1.0 {
|
||||
fmt.Printf(" 🔴 HIGH race condition rate detected!\n")
|
||||
} else if raceRate > 0.1 {
|
||||
fmt.Printf(" 🟡 MODERATE race condition rate\n")
|
||||
} else {
|
||||
fmt.Printf(" 🟢 LOW race condition rate\n")
|
||||
}
|
||||
} else {
|
||||
fmt.Printf(" No race conditions detected\n")
|
||||
if stats.errors == 0 {
|
||||
fmt.Printf(" 🟢 All operations completed successfully\n")
|
||||
}
|
||||
}
|
||||
|
||||
if stats.errors > 0 {
|
||||
errorRate := float64(stats.errors) / float64(totalOps) * 100
|
||||
fmt.Printf(" Overall error rate: %.2f%%\n", errorRate)
|
||||
}
|
||||
|
||||
fmt.Println(strings.Repeat("=", 60))
|
||||
|
||||
// Recommendations
|
||||
if stats.raceConditions > 0 || stats.errors > totalOps/10 {
|
||||
fmt.Println("\nRecommendations:")
|
||||
if stats.raceConditions > 0 {
|
||||
fmt.Println(" • Race conditions detected - investigate filer concurrent access handling")
|
||||
fmt.Println(" • Check filer logs for 'leveldb: closed' or 'transport is closing' errors")
|
||||
}
|
||||
if stats.errors > totalOps/20 {
|
||||
fmt.Println(" • High error rate - check filer stability and resource limits")
|
||||
}
|
||||
fmt.Println(" • Consider running with -verbose flag for detailed error analysis")
|
||||
}
|
||||
}
|
||||
@@ -6,20 +6,16 @@ services:
|
||||
command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
interval: 1s
|
||||
timeout: 60s
|
||||
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 volume -master=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
|
||||
command: "-v=4 volume -mserver=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz" ]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 5s
|
||||
interval: 1s
|
||||
timeout: 30s
|
||||
depends_on:
|
||||
master:
|
||||
condition: service_healthy
|
||||
@@ -29,10 +25,8 @@ services:
|
||||
command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8888" ]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 5s
|
||||
interval: 1s
|
||||
timeout: 30s
|
||||
depends_on:
|
||||
volume:
|
||||
condition: service_healthy
|
||||
@@ -52,10 +46,8 @@ services:
|
||||
memory: 4096m
|
||||
healthcheck:
|
||||
test: [ "CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs" ]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 10s
|
||||
interval: 1s
|
||||
timeout: 30s
|
||||
depends_on:
|
||||
filer:
|
||||
condition: service_healthy
|
||||
|
||||
@@ -36,7 +36,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -46,7 +46,7 @@ services:
|
||||
ports:
|
||||
- 8082:8082
|
||||
- 18082:18082
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -56,7 +56,7 @@ services:
|
||||
ports:
|
||||
- 8083:8083
|
||||
- 18083:18083
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
|
||||
@@ -36,7 +36,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -46,7 +46,7 @@ services:
|
||||
ports:
|
||||
- 8082:8082
|
||||
- 18082:18082
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -56,7 +56,7 @@ services:
|
||||
ports:
|
||||
- 8083:8083
|
||||
- 18083:18083
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
|
||||
@@ -16,7 +16,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "-v=1 volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1 -max=10000"
|
||||
command: "-v=1 volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1 -max=10000"
|
||||
depends_on:
|
||||
- master
|
||||
volumes:
|
||||
|
||||
@@ -42,7 +42,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -51,7 +51,7 @@ services:
|
||||
ports:
|
||||
- 8082:8082
|
||||
- 18082:18082
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
@@ -60,7 +60,7 @@ services:
|
||||
ports:
|
||||
- 8083:8083
|
||||
- 18083:18083
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume"
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume"
|
||||
depends_on:
|
||||
- master
|
||||
mysql:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 7455:8080
|
||||
- 9325:9325
|
||||
command: 'volume -master="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
- 9325:9325
|
||||
volumes:
|
||||
- /Volumes/mobile_disk/99:/data
|
||||
command: 'volume -master="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume"
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
|
||||
@@ -15,7 +15,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
@@ -24,7 +24,7 @@ services:
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
- 8000:8000
|
||||
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false'
|
||||
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false'
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
depends_on:
|
||||
|
||||
@@ -10,10 +10,10 @@
|
||||
scripts = """
|
||||
lock
|
||||
ec.encode -fullPercent=95 -quietFor=1h
|
||||
ec.rebuild -apply
|
||||
ec.balance -apply
|
||||
volume.balance -apply
|
||||
volume.fix.replication -apply
|
||||
ec.rebuild -force
|
||||
ec.balance -force
|
||||
volume.balance -force
|
||||
volume.fix.replication
|
||||
unlock
|
||||
"""
|
||||
sleep_minutes = 17 # sleep minutes between each script execution
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
# 2021-01-30 16:25:30
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
|
||||
etcd:
|
||||
@@ -71,7 +73,7 @@ services:
|
||||
- net
|
||||
command:
|
||||
- 'volume'
|
||||
- '-master=master:9333'
|
||||
- '-mserver=master:9333'
|
||||
- '-port=8080'
|
||||
deploy:
|
||||
mode: global
|
||||
|
||||
@@ -17,7 +17,7 @@ services:
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
|
||||
depends_on:
|
||||
- master
|
||||
s3:
|
||||
|
||||
@@ -15,7 +15,7 @@ services:
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "server -ip=127.0.0.1 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false"
|
||||
command: "server -ip=127.0.0.1 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false"
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
environment:
|
||||
|
||||
@@ -20,7 +20,7 @@ services:
|
||||
- 8888:8888
|
||||
- 8000:8000
|
||||
- 18888:18888
|
||||
command: "server -ip=s3 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false"
|
||||
command: "server -ip=s3 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false"
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
environment:
|
||||
|
||||
+2
-34
@@ -1,37 +1,5 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Enable FIPS 140-3 mode by default (Go 1.24+)
|
||||
# To disable: docker run -e GODEBUG=fips140=off ...
|
||||
export GODEBUG="${GODEBUG:+$GODEBUG,}fips140=on"
|
||||
|
||||
# Fix permissions for mounted volumes
|
||||
# If /data is mounted from host, it might have different ownership
|
||||
# Fix this by ensuring seaweed user owns the directory
|
||||
if [ "$(id -u)" = "0" ]; then
|
||||
# Running as root, check and fix permissions if needed
|
||||
SEAWEED_UID=$(id -u seaweed)
|
||||
SEAWEED_GID=$(id -g seaweed)
|
||||
|
||||
# Verify seaweed user and group exist
|
||||
if [ -z "$SEAWEED_UID" ] || [ -z "$SEAWEED_GID" ]; then
|
||||
echo "Error: 'seaweed' user or group not found. Cannot fix permissions." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DATA_UID=$(stat -c '%u' /data 2>/dev/null)
|
||||
DATA_GID=$(stat -c '%g' /data 2>/dev/null)
|
||||
|
||||
# Only run chown -R if ownership doesn't match (much faster for subsequent starts)
|
||||
echo "Fixing /data ownership for seaweed user (uid=$SEAWEED_UID, gid=$SEAWEED_GID)"
|
||||
if ! chown -R seaweed:seaweed /data; then
|
||||
echo "Warning: Failed to change ownership of /data. This may cause permission errors." >&2
|
||||
echo "If /data is read-only or has mount issues, the application may fail to start." >&2
|
||||
fi
|
||||
|
||||
# Use su-exec to drop privileges and run as seaweed user
|
||||
exec su-exec seaweed "$0" "$@"
|
||||
fi
|
||||
|
||||
isArgPassed() {
|
||||
arg="$1"
|
||||
argWithEqualSign="$1="
|
||||
@@ -40,10 +8,10 @@ isArgPassed() {
|
||||
passedArg="$1"
|
||||
shift
|
||||
case $passedArg in
|
||||
"$arg")
|
||||
$arg)
|
||||
return 0
|
||||
;;
|
||||
"$argWithEqualSign"*)
|
||||
$argWithEqualSign*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Simplified entrypoint for e2e testing
|
||||
#
|
||||
# This script intentionally runs as root for e2e test environments to:
|
||||
# 1. Simplify test setup and avoid permission-related test failures
|
||||
# 2. Eliminate dependency on Alpine-specific tools (su-exec) since we use Ubuntu base
|
||||
# 3. Focus testing on application logic rather than container security
|
||||
#
|
||||
# IMPORTANT: Production deployments should use Dockerfile.go_build with proper
|
||||
# user isolation via su-exec. This simplified approach is ONLY for testing.
|
||||
|
||||
isArgPassed() {
|
||||
arg="$1"
|
||||
argWithEqualSign="$1="
|
||||
shift
|
||||
while [ $# -gt 0 ]; do
|
||||
passedArg="$1"
|
||||
shift
|
||||
case $passedArg in
|
||||
"$arg")
|
||||
return 0
|
||||
;;
|
||||
"$argWithEqualSign"*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
|
||||
'master')
|
||||
ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true master $ARGS "$@"
|
||||
;;
|
||||
|
||||
'volume')
|
||||
ARGS="-dir=/data -max=0"
|
||||
if isArgPassed "-max" "$@"; then
|
||||
ARGS="-dir=/data"
|
||||
fi
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true volume $ARGS "$@"
|
||||
;;
|
||||
|
||||
'server')
|
||||
ARGS="-dir=/data -volume.max=0 -master.volumePreallocate -master.volumeSizeLimitMB=1024"
|
||||
if isArgPassed "-volume.max" "$@"; then
|
||||
ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
|
||||
fi
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true server $ARGS "$@"
|
||||
;;
|
||||
|
||||
'filer')
|
||||
ARGS=""
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true filer $ARGS "$@"
|
||||
;;
|
||||
|
||||
's3')
|
||||
ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true s3 $ARGS "$@"
|
||||
;;
|
||||
|
||||
'mount')
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true mount "$@"
|
||||
;;
|
||||
|
||||
'shell')
|
||||
ARGS="-cluster=$SHELL_CLUSTER -filer=$SHELL_FILER -filerGroup=$SHELL_FILER_GROUP -master=$SHELL_MASTER -options=$SHELL_OPTIONS"
|
||||
shift
|
||||
exec echo "$@" | /usr/bin/weed -logtostderr=true shell $ARGS
|
||||
;;
|
||||
|
||||
*)
|
||||
exec /usr/bin/weed "$@"
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
[filer.options]
|
||||
# with http DELETE, by default the filer would check whether a folder is empty.
|
||||
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
|
||||
recursive_delete = false
|
||||
|
||||
####################################################
|
||||
# FoundationDB store
|
||||
####################################################
|
||||
[foundationdb]
|
||||
enabled = true
|
||||
cluster_file = "/etc/foundationdb/fdb.cluster"
|
||||
api_version = 740
|
||||
# Optional: timeout for FDB operations (default: 10s)
|
||||
# timeout = "10s"
|
||||
# Optional: max retry delay for retryable errors (default: 1s)
|
||||
# max_retry_delay = "1s"
|
||||
# Optional: directory prefix for storing SeaweedFS data (default: "seaweedfs")
|
||||
# directory_prefix = "seaweedfs"
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Helper script to get SHA256 checksum for FoundationDB client package
|
||||
# Usage: ./get_fdb_checksum.sh <version> [arch]
|
||||
# Example: ./get_fdb_checksum.sh 7.4.5 amd64
|
||||
# Example: ./get_fdb_checksum.sh 7.4.5 arm64
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ $# -lt 1 ] || [ $# -gt 2 ]; then
|
||||
echo "Usage: $0 <fdb_version> [arch]" >&2
|
||||
echo "Example: $0 7.4.5" >&2
|
||||
echo "Example: $0 7.4.5 arm64" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FDB_VERSION="$1"
|
||||
FDB_ARCH="${2:-amd64}"
|
||||
|
||||
case "$FDB_ARCH" in
|
||||
"amd64")
|
||||
CANONICAL_ARCH="amd64"
|
||||
PACKAGE_ARCH="amd64"
|
||||
;;
|
||||
"arm64"|"aarch64")
|
||||
CANONICAL_ARCH="arm64"
|
||||
PACKAGE_ARCH="aarch64"
|
||||
;;
|
||||
*)
|
||||
echo "Error: Architecture must be 'amd64', 'arm64', or 'aarch64'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb"
|
||||
URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}"
|
||||
|
||||
echo "Downloading FoundationDB ${FDB_VERSION} client package for ${FDB_ARCH}..."
|
||||
echo "URL: ${URL}"
|
||||
echo ""
|
||||
|
||||
# Download to temp directory
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "${TEMP_DIR}"' EXIT
|
||||
|
||||
cd "${TEMP_DIR}"
|
||||
if wget --timeout=30 --tries=3 -q "${URL}"; then
|
||||
CHECKSUM=$(sha256sum "${PACKAGE}" | awk '{print $1}')
|
||||
echo "✓ Download successful"
|
||||
echo ""
|
||||
echo "SHA256 Checksum:"
|
||||
echo "${CHECKSUM}"
|
||||
echo ""
|
||||
echo "Add this to Dockerfile.foundationdb_large:"
|
||||
echo " \"${FDB_VERSION}_${CANONICAL_ARCH}\") \\"
|
||||
echo " EXPECTED_SHA256=\"${CHECKSUM}\" ;; \\"
|
||||
else
|
||||
echo "✗ Failed to download package from ${URL}" >&2
|
||||
echo "Please verify the version number, architecture, and URL" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -7,14 +7,14 @@ services:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
- 9324:9324
|
||||
command: 'master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324'
|
||||
command: "master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324"
|
||||
volume:
|
||||
image: chrislusf/seaweedfs # use a remote image
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
- 9325:9325
|
||||
command: 'volume -ip=volume -master="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325'
|
||||
command: 'volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
@@ -23,7 +23,7 @@ services:
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
- 9326:9326
|
||||
command: 'filer -ip=filer -master="master:9333" -ip.bind=0.0.0.0 -metricsPort=9326'
|
||||
command: 'filer -master="master:9333" -ip.bind=0.0.0.0 -metricsPort=9326'
|
||||
tty: true
|
||||
stdin_open: true
|
||||
depends_on:
|
||||
@@ -54,6 +54,6 @@ services:
|
||||
- 9000:9090
|
||||
volumes:
|
||||
- ./prometheus:/etc/prometheus
|
||||
command: '--web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml'
|
||||
command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
|
||||
depends_on:
|
||||
- s3
|
||||
|
||||
@@ -6,13 +6,13 @@ services:
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: 'master -ip=master -ip.bind=0.0.0.0'
|
||||
command: "master -ip=master"
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:dev # use a remote dev image
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -ip=volume -master="master:9333" -ip.bind=0.0.0.0 -port=8080'
|
||||
command: 'volume -mserver="master:9333" -port=8080 -ip=volume'
|
||||
depends_on:
|
||||
- master
|
||||
filer:
|
||||
@@ -20,7 +20,7 @@ services:
|
||||
ports:
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
command: 'filer -ip=filer -master="master:9333" -ip.bind=0.0.0.0'
|
||||
command: 'filer -master="master:9333" -ip.bind=0.0.0.0'
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.24.9
|
||||
go 1.24
|
||||
|
||||
toolchain go1.24.1
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.121.6 // indirect
|
||||
cloud.google.com/go/pubsub v1.50.1
|
||||
cloud.google.com/go/storage v1.57.1
|
||||
cloud.google.com/go/pubsub v1.50.0
|
||||
cloud.google.com/go/storage v1.56.1
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -17,8 +21,8 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/eapache/go-resiliency v1.6.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
|
||||
github.com/eapache/go-resiliency v1.3.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
|
||||
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
|
||||
@@ -26,33 +30,34 @@ require (
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.15.0
|
||||
github.com/go-redsync/redsync/v4 v4.13.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/gocql/gocql v1.7.0
|
||||
github.com/golang/protobuf v1.5.4
|
||||
github.com/golang/snappy v1.0.0
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/google/wire v0.6.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/jackc/pgx/v5 v5.7.6
|
||||
github.com/jackc/pgx/v5 v5.7.5
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.2
|
||||
github.com/klauspost/reedsolomon v1.12.6
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.12.5
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.3
|
||||
github.com/linxGnu/grocksdb v1.10.2
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.11 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -62,23 +67,23 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_golang v1.23.0
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.19.2
|
||||
github.com/prometheus/common v0.65.0 // indirect
|
||||
github.com/prometheus/procfs v0.17.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.6
|
||||
github.com/seaweedfs/raft v1.1.3
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/spf13/afero v1.12.0 // indirect
|
||||
github.com/spf13/cast v1.7.1 // indirect
|
||||
github.com/spf13/viper v1.20.1
|
||||
github.com/stretchr/testify v1.11.0
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.18.0
|
||||
github.com/tidwall/match v1.2.0
|
||||
github.com/tidwall/match v1.1.1
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
|
||||
github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
|
||||
@@ -88,80 +93,73 @@ require (
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.6
|
||||
go.mongodb.org/mongo-driver v1.17.6
|
||||
go.etcd.io/etcd/client/v3 v3.6.4
|
||||
go.mongodb.org/mongo-driver v1.17.4
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.44.0
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0
|
||||
golang.org/x/crypto v0.46.0
|
||||
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
|
||||
golang.org/x/image v0.34.0
|
||||
golang.org/x/net v0.48.0
|
||||
golang.org/x/oauth2 v0.34.0
|
||||
golang.org/x/sys v0.39.0
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
gocloud.dev v0.43.0
|
||||
gocloud.dev/pubsub/natspubsub v0.43.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.43.0
|
||||
golang.org/x/crypto v0.41.0
|
||||
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b
|
||||
golang.org/x/image v0.30.0
|
||||
golang.org/x/net v0.43.0
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/sys v0.35.0
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
golang.org/x/tools v0.36.0
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.258.0
|
||||
google.golang.org/api v0.247.0
|
||||
google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
|
||||
google.golang.org/grpc v1.77.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
google.golang.org/grpc v1.75.0
|
||||
google.golang.org/protobuf v1.36.8
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.42.2
|
||||
modernc.org/sqlite v1.38.2
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.23.2
|
||||
cloud.google.com/go/kms v1.22.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.943
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/ThreeDotsLabs/watermill v1.5.0
|
||||
github.com/a-h/templ v0.3.924
|
||||
github.com/arangodb/go-driver v1.6.6
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.38.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.7
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.40.0
|
||||
github.com/getsentry/sentry-go v0.35.0
|
||||
github.com/gin-contrib/sessions v1.0.4
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/gin-gonic/gin v1.10.1
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hanwen/go-fuse/v2 v2.8.0
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.17.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/linkedin/goavro/v2 v2.14.1
|
||||
github.com/mattn/go-sqlite3 v1.14.32
|
||||
github.com/hashicorp/vault/api v1.20.0
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.26.3
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/parquet-go/parquet-go v0.25.1
|
||||
github.com/pkg/sftp v1.13.9
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.71.2
|
||||
github.com/rclone/rclone v1.70.3
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1
|
||||
github.com/shirou/gopsutil/v4 v4.25.11
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.1
|
||||
github.com/redis/go-redis/v9 v9.12.1
|
||||
github.com/schollz/progressbar/v3 v3.18.0
|
||||
github.com/shirou/gopsutil/v3 v3.24.5
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.113.5
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.4
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
golang.org/x/sync v0.16.0
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
)
|
||||
|
||||
@@ -169,82 +167,41 @@ require github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // ind
|
||||
|
||||
require (
|
||||
cloud.google.com/go/longrunning v0.6.7 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.0.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/bufbuild/protocompile v0.14.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cockroachdb/apd/v3 v3.1.0 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 // indirect
|
||||
github.com/dave/dst v0.27.2 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.2 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jaegertracing/jaeger v1.47.0 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
github.com/parquet-go/bitpack v1.0.0 // indirect
|
||||
github.com/parquet-go/jsonlite v1.0.0 // indirect
|
||||
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/twpayne/go-geom v1.4.1 // indirect
|
||||
github.com/twpayne/go-kml v1.5.2 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
gonum.org/v1/gonum v0.16.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.24.0 // indirect
|
||||
cloud.google.com/go/auth v0.17.0 // indirect
|
||||
cloud.google.com/go/auth v0.16.5 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.8.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.2 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.2 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.11.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.1 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.218 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.173 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.20.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
@@ -255,72 +212,72 @@ require (
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.10.3 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 // indirect
|
||||
github.com/aws/smithy-go v1.22.5 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/bytedance/sonic v1.14.0 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
github.com/bytedance/sonic v1.13.2 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.12.0 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.10.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/ebitengine/purego v0.9.1 // indirect
|
||||
github.com/elastic/gosigar v0.14.3 // indirect
|
||||
github.com/ebitengine/purego v0.8.4 // indirect
|
||||
github.com/elastic/gosigar v0.14.2 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/fatih/color v1.16.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.9 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/gin-contrib/sse v1.0.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.2 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.2 // indirect
|
||||
github.com/go-openapi/errors v0.22.1 // indirect
|
||||
github.com/go-openapi/strfmt v0.23.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.27.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.26.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.16.5 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
@@ -328,19 +285,19 @@ require (
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/gorilla/context v1.1.2 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-metrics v0.5.4 // indirect
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
|
||||
github.com/henrybear327/go-proton-api v1.0.0 // indirect
|
||||
@@ -354,12 +311,12 @@ require (
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/k0kubun/pp v3.0.1+incompatible
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lanrat/extsort v1.4.0 // indirect
|
||||
github.com/lanrat/extsort v1.0.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
|
||||
@@ -367,7 +324,7 @@ require (
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.43.0 // indirect
|
||||
@@ -379,19 +336,19 @@ require (
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.98.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.93.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/pkg/xattr v0.4.12 // indirect
|
||||
github.com/pkg/xattr v0.4.10 // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
@@ -399,65 +356,66 @@ require (
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/samber/lo v1.51.0 // indirect
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138
|
||||
github.com/sagikazarmark/locafero v0.7.0 // indirect
|
||||
github.com/samber/lo v1.50.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.25.5 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/smartystreets/goconvey v1.8.1 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.3.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.15 // indirect
|
||||
github.com/tklauser/numcpus v0.10.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/ugorji/go/codec v1.3.0 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
github.com/unknwon/goconfig v1.0.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20241112172322-ea1f63298f77 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.2 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.6 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.0 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.37.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
|
||||
go.opentelemetry.io/otel v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.37.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/term v0.38.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
|
||||
golang.org/x/arch v0.16.0 // indirect
|
||||
golang.org/x/term v0.34.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.66.10 // indirect
|
||||
modernc.org/libc v1.66.3 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20250808122759-804533d519c1 // indirect
|
||||
sigs.k8s.io/yaml v1.4.0 // indirect
|
||||
storj.io/common v0.0.0-20250605163628-70ca83b6228e // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.06"
|
||||
appVersion: "3.96"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.0.406
|
||||
version: 4.0.396
|
||||
|
||||
@@ -145,191 +145,6 @@ stringData:
|
||||
seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"snu8yoP6QAlY0ne4","secretKey":"PNzBcmeLNEdR0oviwm04NQAicOrDH1Km"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"SCigFee6c5lbi04A","secretKey":"kgFhbT38R8WUYVtiFQ1OiSVOrYr3NKku"}],"actions":["Read"]}]}'
|
||||
```
|
||||
|
||||
## Admin Component
|
||||
|
||||
The admin component provides a modern web-based administration interface for managing SeaweedFS clusters. It includes:
|
||||
|
||||
- **Dashboard**: Real-time cluster status and metrics
|
||||
- **Volume Management**: Monitor volume servers, capacity, and health
|
||||
- **File Browser**: Browse and manage files in the filer
|
||||
- **Maintenance Operations**: Trigger maintenance tasks via workers
|
||||
- **Object Store Management**: Create and manage buckets with web interface
|
||||
|
||||
### Enabling Admin
|
||||
|
||||
To enable the admin interface, add the following to your values.yaml:
|
||||
|
||||
```yaml
|
||||
admin:
|
||||
enabled: true
|
||||
port: 23646
|
||||
grpcPort: 33646 # For worker connections
|
||||
adminUser: "admin"
|
||||
adminPassword: "your-secure-password" # Leave empty to disable auth
|
||||
|
||||
# Optional: persist admin data
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "10Gi"
|
||||
storageClass: "your-storage-class"
|
||||
|
||||
# Optional: enable ingress
|
||||
ingress:
|
||||
enabled: true
|
||||
host: "admin.seaweedfs.local"
|
||||
className: "nginx"
|
||||
```
|
||||
|
||||
The admin interface will be available at `http://<admin-service>:23646` (or via ingress). Workers connect to the admin server via gRPC on port `33646`.
|
||||
|
||||
### Admin Authentication
|
||||
|
||||
If `adminPassword` is set, the admin interface requires authentication:
|
||||
- Username: Value of `adminUser` (default: `admin`)
|
||||
- Password: Value of `adminPassword`
|
||||
|
||||
If `adminPassword` is empty or not set, the admin interface runs without authentication (not recommended for production).
|
||||
|
||||
### Admin Data Persistence
|
||||
|
||||
The admin component can store configuration and maintenance data. You can configure storage in several ways:
|
||||
|
||||
- **emptyDir** (default): Data is lost when pod restarts
|
||||
- **persistentVolumeClaim**: Data persists across pod restarts
|
||||
- **hostPath**: Data stored on the host filesystem
|
||||
- **existingClaim**: Use an existing PVC
|
||||
|
||||
## Worker Component
|
||||
|
||||
Workers are maintenance agents that execute cluster maintenance tasks such as vacuum, volume balancing, and erasure coding. Workers connect to the admin server via gRPC and receive task assignments.
|
||||
|
||||
### Enabling Workers
|
||||
|
||||
To enable workers, add the following to your values.yaml:
|
||||
|
||||
```yaml
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 2 # Scale based on workload
|
||||
capabilities: "vacuum,balance,erasure_coding" # Tasks this worker can handle
|
||||
maxConcurrent: 3 # Maximum concurrent tasks per worker
|
||||
|
||||
# Working directory for task execution
|
||||
# Default: "/tmp/seaweedfs-worker"
|
||||
# Note: /tmp is ephemeral - use persistent storage (hostPath/existingClaim) for long-running tasks
|
||||
workingDir: "/tmp/seaweedfs-worker"
|
||||
|
||||
# Optional: configure admin server address
|
||||
# If not specified, auto-discovers from admin service in the same namespace by looking for
|
||||
# a service named "<release-name>-admin" (e.g., "seaweedfs-admin").
|
||||
# Auto-discovery only works if the admin is in the same namespace and same Helm release.
|
||||
# For cross-namespace or separate release scenarios, explicitly set this value.
|
||||
# Example: If main SeaweedFS is deployed in "production" namespace:
|
||||
# adminServer: "seaweedfs-admin.production.svc:33646"
|
||||
adminServer: ""
|
||||
|
||||
# Workers need storage for task execution
|
||||
# Note: Workers use a Deployment, which does not support `volumeClaimTemplates`
|
||||
# for dynamic PVC creation per pod. To use persistent storage, you must
|
||||
# pre-provision a PersistentVolumeClaim and use `type: "existingClaim"`.
|
||||
data:
|
||||
type: "emptyDir" # Options: "emptyDir", "hostPath", or "existingClaim"
|
||||
hostPathPrefix: /storage # For hostPath
|
||||
# claimName: "worker-pvc" # For existingClaim with pre-provisioned PVC
|
||||
|
||||
# Resource limits for worker pods
|
||||
resources:
|
||||
requests:
|
||||
cpu: "500m"
|
||||
memory: "512Mi"
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: "2Gi"
|
||||
```
|
||||
|
||||
### Worker Capabilities
|
||||
|
||||
Workers can be configured with different capabilities:
|
||||
- **vacuum**: Reclaim deleted file space
|
||||
- **balance**: Balance volumes across volume servers
|
||||
- **erasure_coding**: Handle erasure coding operations
|
||||
|
||||
You can configure workers with all capabilities or create specialized worker pools with specific capabilities.
|
||||
|
||||
### Worker Deployment Strategy
|
||||
|
||||
For production deployments, consider:
|
||||
|
||||
1. **Multiple Workers**: Deploy 2+ worker replicas for high availability
|
||||
2. **Resource Allocation**: Workers need sufficient CPU/memory for maintenance tasks
|
||||
3. **Storage**: Workers need temporary storage for vacuum and balance operations (size depends on volume size)
|
||||
4. **Specialized Workers**: Create separate worker deployments for different capabilities if needed
|
||||
|
||||
Example specialized worker configuration:
|
||||
|
||||
For specialized worker pools, deploy separate Helm releases with different capabilities:
|
||||
|
||||
**values-worker-vacuum.yaml** (for vacuum operations):
|
||||
```yaml
|
||||
# Disable all other components, enable only workers
|
||||
master:
|
||||
enabled: false
|
||||
volume:
|
||||
enabled: false
|
||||
filer:
|
||||
enabled: false
|
||||
s3:
|
||||
enabled: false
|
||||
admin:
|
||||
enabled: false
|
||||
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 2
|
||||
capabilities: "vacuum"
|
||||
maxConcurrent: 2
|
||||
# REQUIRED: Point to the admin service of your main SeaweedFS release
|
||||
# Replace <namespace> with the namespace where your main seaweedfs is deployed
|
||||
# Example: If deploying in namespace "production":
|
||||
# adminServer: "seaweedfs-admin.production.svc:33646"
|
||||
adminServer: "seaweedfs-admin.<namespace>.svc:33646"
|
||||
```
|
||||
|
||||
**values-worker-balance.yaml** (for balance operations):
|
||||
```yaml
|
||||
# Disable all other components, enable only workers
|
||||
master:
|
||||
enabled: false
|
||||
volume:
|
||||
enabled: false
|
||||
filer:
|
||||
enabled: false
|
||||
s3:
|
||||
enabled: false
|
||||
admin:
|
||||
enabled: false
|
||||
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
capabilities: "balance"
|
||||
maxConcurrent: 1
|
||||
# REQUIRED: Point to the admin service of your main SeaweedFS release
|
||||
# Replace <namespace> with the namespace where your main seaweedfs is deployed
|
||||
# Example: If deploying in namespace "production":
|
||||
# adminServer: "seaweedfs-admin.production.svc:33646"
|
||||
adminServer: "seaweedfs-admin.<namespace>.svc:33646"
|
||||
```
|
||||
|
||||
Deploy the specialized workers as separate releases:
|
||||
```bash
|
||||
# Deploy vacuum workers
|
||||
helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum.yaml
|
||||
|
||||
# Deploy balance workers
|
||||
helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml
|
||||
```
|
||||
|
||||
## Enterprise
|
||||
|
||||
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
|
||||
|
||||
@@ -1595,101 +1595,6 @@
|
||||
"title": "S3 Bucket Traffic Sent",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "reqps",
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 41
|
||||
},
|
||||
"id": 86,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -1754,8 +1659,8 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 41
|
||||
},
|
||||
"id": 72,
|
||||
@@ -3361,209 +3266,6 @@
|
||||
],
|
||||
"title": "Filer Go Routines",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 48
|
||||
},
|
||||
"id": 89,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Size",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 48
|
||||
},
|
||||
"id": 90,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Object Count",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
@@ -3654,4 +3356,4 @@
|
||||
"uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
{{- if and .Values.admin.enabled .Values.admin.ingress.enabled }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
{{- else }}
|
||||
apiVersion: extensions/v1beta1
|
||||
{{- end }}
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-{{ template "seaweedfs.name" . }}-admin
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
{{- if and (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) .Values.admin.ingress.className }}
|
||||
kubernetes.io/ingress.class: {{ .Values.admin.ingress.className }}
|
||||
{{- end }}
|
||||
{{- with .Values.admin.ingress.annotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
spec:
|
||||
{{- if and (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) .Values.admin.ingress.className }}
|
||||
ingressClassName: {{ .Values.admin.ingress.className | quote }}
|
||||
{{- end }}
|
||||
tls:
|
||||
{{ .Values.admin.ingress.tls | default list | toYaml | nindent 6}}
|
||||
rules:
|
||||
- {{- if .Values.admin.ingress.host }}
|
||||
host: {{ .Values.admin.ingress.host | quote }}
|
||||
{{- end }}
|
||||
http:
|
||||
paths:
|
||||
- path: {{ .Values.admin.ingress.path | quote }}
|
||||
{{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
pathType: {{ .Values.admin.ingress.pathType | quote }}
|
||||
{{- end }}
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ template "seaweedfs.name" . }}-admin
|
||||
port:
|
||||
number: {{ .Values.admin.port }}
|
||||
{{- else }}
|
||||
serviceName: {{ template "seaweedfs.name" . }}-admin
|
||||
servicePort: {{ .Values.admin.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,20 +0,0 @@
|
||||
{{- if and .Values.admin.enabled .Values.admin.secret.adminPassword (not .Values.admin.secret.existingSecret) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
type: Opaque
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-admin-secret
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
"helm.sh/resource-policy": keep
|
||||
"helm.sh/hook": "pre-install,pre-upgrade"
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
data:
|
||||
adminUser: {{ .Values.admin.secret.adminUser | b64enc }}
|
||||
adminPassword: {{ .Values.admin.secret.adminPassword | b64enc }}
|
||||
{{- end}}
|
||||
@@ -1,33 +0,0 @@
|
||||
{{- if .Values.admin.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-admin
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{- if .Values.admin.service.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.admin.service.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.admin.service.type }}
|
||||
ports:
|
||||
- name: "http"
|
||||
port: {{ .Values.admin.port }}
|
||||
targetPort: {{ .Values.admin.port }}
|
||||
protocol: TCP
|
||||
- name: "grpc"
|
||||
port: {{ .Values.admin.grpcPort }}
|
||||
targetPort: {{ .Values.admin.grpcPort }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{- end }}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
{{- if .Values.admin.enabled }}
|
||||
{{- if .Values.global.monitoring.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-admin
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{- with .Values.global.monitoring.additionalLabels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.admin.serviceMonitor.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
port: http
|
||||
scrapeTimeout: 5s
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,341 +0,0 @@
|
||||
{{- if .Values.admin.enabled }}
|
||||
{{- if and (not .Values.admin.masters) (not .Values.global.masterServer) (not .Values.master.enabled) }}
|
||||
{{- fail "admin.masters or global.masterServer must be set if master.enabled is false" -}}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-admin
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{- if .Values.admin.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.admin.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceName: {{ template "seaweedfs.name" . }}-admin
|
||||
podManagementPolicy: {{ .Values.admin.podManagementPolicy }}
|
||||
replicas: {{ .Values.admin.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{ with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.admin.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{ with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.admin.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.restartPolicy .Values.admin.restartPolicy }}
|
||||
{{- if .Values.admin.affinity }}
|
||||
affinity:
|
||||
{{ tpl .Values.admin.affinity . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.topologySpreadConstraints }}
|
||||
topologySpreadConstraints:
|
||||
{{ tpl .Values.admin.topologySpreadConstraints . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.tolerations }}
|
||||
tolerations:
|
||||
{{ tpl .Values.admin.tolerations . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
terminationGracePeriodSeconds: 30
|
||||
{{- if .Values.admin.priorityClassName }}
|
||||
priorityClassName: {{ .Values.admin.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.admin.serviceAccountName }}
|
||||
serviceAccountName: {{ .Values.admin.serviceAccountName | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.initContainers }}
|
||||
initContainers:
|
||||
{{ tpl .Values.admin.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.podSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.admin.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "admin.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
|
||||
{{- $adminAuthEnabled := or .Values.admin.secret.existingSecret .Values.admin.secret.adminPassword }}
|
||||
{{- if and .Values.admin.secret.existingSecret (not .Values.admin.secret.userKey) -}}
|
||||
{{- fail "admin.secret.userKey must be set when admin.secret.existingSecret is provided" -}}
|
||||
{{- end -}}
|
||||
{{- if and .Values.admin.secret.existingSecret (not .Values.admin.secret.pwKey) -}}
|
||||
{{- fail "admin.secret.pwKey must be set when admin.secret.existingSecret is provided" -}}
|
||||
{{- end -}}
|
||||
{{- $adminSecretName := .Values.admin.secret.existingSecret | default (printf "%s-admin-secret" (include "seaweedfs.name" .)) }}
|
||||
env:
|
||||
{{- if $adminAuthEnabled }}
|
||||
- name: SEAWEEDFS_ADMIN_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $adminSecretName }}
|
||||
key: {{ if .Values.admin.secret.existingSecret }}{{ .Values.admin.secret.userKey }}{{ else }}adminUser{{ end }}
|
||||
- name: SEAWEEDFS_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $adminSecretName }}
|
||||
key: {{ if .Values.admin.secret.existingSecret }}{{ .Values.admin.secret.pwKey }}{{ else }}adminPassword{{ end }}
|
||||
{{- end }}
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ template "seaweedfs.name" . }}"
|
||||
{{- if .Values.admin.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.admin.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
exec /usr/bin/weed \
|
||||
{{- if or (eq .Values.admin.logs.type "hostPath") (eq .Values.admin.logs.type "persistentVolumeClaim") (eq .Values.admin.logs.type "emptyDir") (eq .Values.admin.logs.type "existingClaim") }}
|
||||
-logdir=/logs \
|
||||
{{- else }}
|
||||
-logtostderr=true \
|
||||
{{- end }}
|
||||
{{- if .Values.admin.loggingOverrideLevel }}
|
||||
-v={{ .Values.admin.loggingOverrideLevel }} \
|
||||
{{- else }}
|
||||
-v={{ .Values.global.loggingLevel }} \
|
||||
{{- end }}
|
||||
admin \
|
||||
-port={{ .Values.admin.port }} \
|
||||
-port.grpc={{ .Values.admin.grpcPort }} \
|
||||
{{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }}
|
||||
-dataDir=/data \
|
||||
{{- else if .Values.admin.dataDir }}
|
||||
-dataDir={{ .Values.admin.dataDir }} \
|
||||
{{- end }}
|
||||
{{- if $adminAuthEnabled }}
|
||||
-adminUser="${SEAWEEDFS_ADMIN_USER}" \
|
||||
-adminPassword="${SEAWEEDFS_ADMIN_PASSWORD}" \
|
||||
{{- end }}
|
||||
{{- if .Values.admin.masters }}
|
||||
-masters={{ .Values.admin.masters }}{{- if .Values.admin.extraArgs }} \{{ end }}
|
||||
{{- else if .Values.global.masterServer }}
|
||||
-masters={{ .Values.global.masterServer }}{{- if .Values.admin.extraArgs }} \{{ end }}
|
||||
{{- else }}
|
||||
-masters={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{- if .Values.admin.extraArgs }} \{{ end }}
|
||||
{{- end }}
|
||||
{{- range $index, $arg := .Values.admin.extraArgs }}
|
||||
{{ $arg }}{{- if lt $index (sub (len $.Values.admin.extraArgs) 1) }} \{{ end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }}
|
||||
- name: admin-data
|
||||
mountPath: /data
|
||||
{{- end }}
|
||||
{{- if or (eq .Values.admin.logs.type "hostPath") (eq .Values.admin.logs.type "persistentVolumeClaim") (eq .Values.admin.logs.type "emptyDir") (eq .Values.admin.logs.type "existingClaim") }}
|
||||
- name: admin-logs
|
||||
mountPath: /logs
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
- name: master-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/master/
|
||||
- name: volume-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/volume/
|
||||
- name: filer-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/filer/
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
- name: admin-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/admin/
|
||||
{{- end }}
|
||||
{{ tpl .Values.admin.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.admin.port }}
|
||||
name: http
|
||||
- containerPort: {{ .Values.admin.grpcPort }}
|
||||
name: grpc
|
||||
{{- if .Values.admin.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: {{ .Values.admin.readinessProbe.httpGet.path }}
|
||||
port: http
|
||||
scheme: {{ .Values.admin.readinessProbe.httpGet.scheme }}
|
||||
initialDelaySeconds: {{ .Values.admin.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.admin.readinessProbe.periodSeconds }}
|
||||
successThreshold: {{ .Values.admin.readinessProbe.successThreshold }}
|
||||
failureThreshold: {{ .Values.admin.readinessProbe.failureThreshold }}
|
||||
timeoutSeconds: {{ .Values.admin.readinessProbe.timeoutSeconds }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: {{ .Values.admin.livenessProbe.httpGet.path }}
|
||||
port: http
|
||||
scheme: {{ .Values.admin.livenessProbe.httpGet.scheme }}
|
||||
initialDelaySeconds: {{ .Values.admin.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.admin.livenessProbe.periodSeconds }}
|
||||
successThreshold: {{ .Values.admin.livenessProbe.successThreshold }}
|
||||
failureThreshold: {{ .Values.admin.livenessProbe.failureThreshold }}
|
||||
timeoutSeconds: {{ .Values.admin.livenessProbe.timeoutSeconds }}
|
||||
{{- end }}
|
||||
{{- with .Values.admin.resources }}
|
||||
resources:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.admin.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.sidecars }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.admin.data.type "hostPath" }}
|
||||
- name: admin-data
|
||||
hostPath:
|
||||
path: {{ .Values.admin.data.hostPathPrefix }}/seaweedfs-admin-data
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if eq .Values.admin.data.type "emptyDir" }}
|
||||
- name: admin-data
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if eq .Values.admin.data.type "existingClaim" }}
|
||||
- name: admin-data
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.admin.data.claimName }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.admin.logs.type "hostPath" }}
|
||||
- name: admin-logs
|
||||
hostPath:
|
||||
path: {{ .Values.admin.logs.hostPathPrefix }}/logs/seaweedfs/admin
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if eq .Values.admin.logs.type "emptyDir" }}
|
||||
- name: admin-logs
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if eq .Values.admin.logs.type "existingClaim" }}
|
||||
- name: admin-logs
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.admin.logs.claimName }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ template "seaweedfs.name" . }}-security-config
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-ca-cert
|
||||
- name: master-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-master-cert
|
||||
- name: volume-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-volume-cert
|
||||
- name: filer-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-filer-cert
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-client-cert
|
||||
- name: admin-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-admin-cert
|
||||
{{- end }}
|
||||
{{ tpl .Values.admin.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.admin.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{ tpl .Values.admin.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "admin.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.admin.data.type "persistentVolumeClaim" }}
|
||||
- metadata:
|
||||
name: admin-data
|
||||
{{- with .Values.admin.data.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 10 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
storageClassName: {{ .Values.admin.data.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.admin.data.size }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.admin.logs.type "persistentVolumeClaim" }}
|
||||
- metadata:
|
||||
name: admin-logs
|
||||
{{- with .Values.admin.logs.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 10 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
storageClassName: {{ .Values.admin.logs.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.admin.logs.size }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -15,9 +15,9 @@ metadata:
|
||||
{{- toYaml .Values.allInOne.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.allInOne.replicas | default 1 }}
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: {{ .Values.allInOne.updateStrategy.type | default "Recreate" }}
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
@@ -79,12 +79,6 @@ spec:
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
|
||||
env:
|
||||
{{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
|
||||
{{- $envMerged := merge (.Values.global.extraEnvironmentVars | default dict) (.Values.allInOne.extraEnvironmentVars | default dict) }}
|
||||
{{- $clusterDefault := default "sw" (index $envMerged "WEED_CLUSTER_DEFAULT") }}
|
||||
{{- $clusterUpper := upper $clusterDefault }}
|
||||
{{- $clusterMasterKey := printf "WEED_CLUSTER_%s_MASTER" $clusterUpper }}
|
||||
{{- $clusterFilerKey := printf "WEED_CLUSTER_%s_FILER" $clusterUpper }}
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
@@ -101,52 +95,32 @@ spec:
|
||||
value: "{{ template "seaweedfs.name" . }}"
|
||||
{{- if .Values.allInOne.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.allInOne.extraEnvironmentVars }}
|
||||
{{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
{{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
# Inject computed cluster endpoints for the default cluster
|
||||
- name: {{ $clusterMasterKey }}
|
||||
value: {{ include "seaweedfs.cluster.masterAddress" . | quote }}
|
||||
- name: {{ $clusterFilerKey }}
|
||||
value: {{ include "seaweedfs.cluster.filerAddress" . | quote }}
|
||||
{{- if .Values.allInOne.secretExtraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.allInOne.secretExtraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
/usr/bin/weed \
|
||||
{{- if .Values.allInOne.loggingOverrideLevel }}
|
||||
-v={{ .Values.allInOne.loggingOverrideLevel }} \
|
||||
{{- else }}
|
||||
-v={{ .Values.global.loggingLevel }} \
|
||||
{{- end }}
|
||||
server \
|
||||
-dir=/data \
|
||||
-master \
|
||||
@@ -202,9 +176,6 @@ spec:
|
||||
{{- else if .Values.master.metricsPort }}
|
||||
-metricsPort={{ .Values.master.metricsPort }} \
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.metricsIp }}
|
||||
-metricsIp={{ .Values.allInOne.metricsIp }} \
|
||||
{{- end }}
|
||||
-filer \
|
||||
-filer.port={{ .Values.filer.port }} \
|
||||
{{- if .Values.filer.disableDirListing }}
|
||||
@@ -233,75 +204,61 @@ spec:
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
-s3 \
|
||||
-s3.port={{ .Values.allInOne.s3.port | default .Values.s3.port }} \
|
||||
{{- $domainName := .Values.allInOne.s3.domainName | default .Values.s3.domainName }}
|
||||
{{- if $domainName }}
|
||||
-s3.domainName={{ $domainName }} \
|
||||
-s3.port={{ .Values.s3.port }} \
|
||||
{{- if .Values.s3.domainName }}
|
||||
-s3.domainName={{ .Values.s3.domainName }} \
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- $httpsPort := .Values.allInOne.s3.httpsPort | default .Values.s3.httpsPort }}
|
||||
{{- if $httpsPort }}
|
||||
-s3.port.https={{ $httpsPort }} \
|
||||
{{- if .Values.s3.httpsPort }}
|
||||
-s3.port.https={{ .Values.s3.httpsPort }} \
|
||||
{{- end }}
|
||||
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end }}
|
||||
{{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
|
||||
{{- if eq (typeOf .Values.s3.allowEmptyFolder) "bool" }}
|
||||
-s3.allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \
|
||||
{{- end }}
|
||||
{{- if .Values.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/s3/seaweedfs_s3_config \
|
||||
{{- end }}
|
||||
{{- $auditLogConfig := .Values.allInOne.s3.auditLogConfig | default .Values.s3.auditLogConfig }}
|
||||
{{- if $auditLogConfig }}
|
||||
{{- if .Values.s3.auditLogConfig }}
|
||||
-s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sftp.enabled }}
|
||||
-sftp \
|
||||
-sftp.port={{ .Values.allInOne.sftp.port | default .Values.sftp.port }} \
|
||||
{{- $sshPrivateKey := .Values.allInOne.sftp.sshPrivateKey | default .Values.sftp.sshPrivateKey }}
|
||||
{{- if $sshPrivateKey }}
|
||||
-sftp.sshPrivateKey={{ $sshPrivateKey }} \
|
||||
-sftp.port={{ .Values.sftp.port }} \
|
||||
{{- if .Values.sftp.sshPrivateKey }}
|
||||
-sftp.sshPrivateKey={{ .Values.sftp.sshPrivateKey }} \
|
||||
{{- end }}
|
||||
{{- $hostKeysFolder := .Values.allInOne.sftp.hostKeysFolder | default .Values.sftp.hostKeysFolder }}
|
||||
{{- if $hostKeysFolder }}
|
||||
-sftp.hostKeysFolder={{ $hostKeysFolder }} \
|
||||
{{- if .Values.sftp.hostKeysFolder }}
|
||||
-sftp.hostKeysFolder={{ .Values.sftp.hostKeysFolder }} \
|
||||
{{- end }}
|
||||
{{- $authMethods := .Values.allInOne.sftp.authMethods | default .Values.sftp.authMethods }}
|
||||
{{- if $authMethods }}
|
||||
-sftp.authMethods={{ $authMethods }} \
|
||||
{{- if .Values.sftp.authMethods }}
|
||||
-sftp.authMethods={{ .Values.sftp.authMethods }} \
|
||||
{{- end }}
|
||||
{{- $maxAuthTries := .Values.allInOne.sftp.maxAuthTries | default .Values.sftp.maxAuthTries }}
|
||||
{{- if $maxAuthTries }}
|
||||
-sftp.maxAuthTries={{ $maxAuthTries }} \
|
||||
{{- if .Values.sftp.maxAuthTries }}
|
||||
-sftp.maxAuthTries={{ .Values.sftp.maxAuthTries }} \
|
||||
{{- end }}
|
||||
{{- $bannerMessage := .Values.allInOne.sftp.bannerMessage | default .Values.sftp.bannerMessage }}
|
||||
{{- if $bannerMessage }}
|
||||
-sftp.bannerMessage="{{ $bannerMessage }}" \
|
||||
{{- if .Values.sftp.bannerMessage }}
|
||||
-sftp.bannerMessage="{{ .Values.sftp.bannerMessage }}" \
|
||||
{{- end }}
|
||||
{{- $loginGraceTime := .Values.allInOne.sftp.loginGraceTime | default .Values.sftp.loginGraceTime }}
|
||||
{{- if $loginGraceTime }}
|
||||
-sftp.loginGraceTime={{ $loginGraceTime }} \
|
||||
{{- if .Values.sftp.loginGraceTime }}
|
||||
-sftp.loginGraceTime={{ .Values.sftp.loginGraceTime }} \
|
||||
{{- end }}
|
||||
{{- $clientAliveInterval := .Values.allInOne.sftp.clientAliveInterval | default .Values.sftp.clientAliveInterval }}
|
||||
{{- if $clientAliveInterval }}
|
||||
-sftp.clientAliveInterval={{ $clientAliveInterval }} \
|
||||
{{- if .Values.sftp.clientAliveInterval }}
|
||||
-sftp.clientAliveInterval={{ .Values.sftp.clientAliveInterval }} \
|
||||
{{- end }}
|
||||
{{- $clientAliveCountMax := .Values.allInOne.sftp.clientAliveCountMax | default .Values.sftp.clientAliveCountMax }}
|
||||
{{- if $clientAliveCountMax }}
|
||||
-sftp.clientAliveCountMax={{ $clientAliveCountMax }} \
|
||||
{{- if .Values.sftp.clientAliveCountMax }}
|
||||
-sftp.clientAliveCountMax={{ .Values.sftp.clientAliveCountMax }} \
|
||||
{{- end }}
|
||||
{{- if or .Values.allInOne.sftp.enableAuth .Values.sftp.enableAuth }}
|
||||
-sftp.userStoreFile=/etc/sw/sftp/seaweedfs_sftp_config \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $extraArgsCount := len .Values.allInOne.extraArgs }}
|
||||
{{- range $i, $arg := .Values.allInOne.extraArgs }}
|
||||
{{ $arg | quote }}{{ if ne (add1 $i) $extraArgsCount }} \{{ end }}
|
||||
{{- end }}
|
||||
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
{{- if and .Values.allInOne.s3.enabled (or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth) }}
|
||||
{{- if and .Values.allInOne.s3.enabled (or .Values.s3.enableAuth .Values.filer.s3.enableAuth) }}
|
||||
- name: config-s3-users
|
||||
mountPath: /etc/sw/s3
|
||||
readOnly: true
|
||||
@@ -310,12 +267,10 @@ spec:
|
||||
- name: config-ssh
|
||||
mountPath: /etc/sw/ssh
|
||||
readOnly: true
|
||||
{{- if or .Values.allInOne.sftp.enableAuth .Values.sftp.enableAuth }}
|
||||
- mountPath: /etc/sw/sftp
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.notificationConfig }}
|
||||
- name: notification-config
|
||||
mountPath: /etc/seaweedfs/notification.toml
|
||||
@@ -362,16 +317,15 @@ spec:
|
||||
- containerPort: {{ .Values.filer.grpcPort }}
|
||||
name: swfs-fil-grpc
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
- containerPort: {{ .Values.allInOne.s3.port | default .Values.s3.port }}
|
||||
- containerPort: {{ .Values.s3.port }}
|
||||
name: swfs-s3
|
||||
{{- $httpsPort := .Values.allInOne.s3.httpsPort | default .Values.s3.httpsPort }}
|
||||
{{- if $httpsPort }}
|
||||
- containerPort: {{ $httpsPort }}
|
||||
{{- if .Values.s3.httpsPort }}
|
||||
- containerPort: {{ .Values.s3.httpsPort }}
|
||||
name: swfs-s3-tls
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sftp.enabled }}
|
||||
- containerPort: {{ .Values.allInOne.sftp.port | default .Values.sftp.port }}
|
||||
- containerPort: {{ .Values.sftp.port }}
|
||||
name: swfs-sftp
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.metricsPort }}
|
||||
@@ -383,7 +337,7 @@ spec:
|
||||
httpGet:
|
||||
path: {{ .Values.allInOne.readinessProbe.httpGet.path }}
|
||||
port: {{ .Values.master.port }}
|
||||
scheme: {{ .Values.allInOne.readinessProbe.httpGet.scheme }}
|
||||
scheme: {{ .Values.allInOne.readinessProbe.scheme }}
|
||||
initialDelaySeconds: {{ .Values.allInOne.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.allInOne.readinessProbe.periodSeconds }}
|
||||
successThreshold: {{ .Values.allInOne.readinessProbe.successThreshold }}
|
||||
@@ -395,7 +349,7 @@ spec:
|
||||
httpGet:
|
||||
path: {{ .Values.allInOne.livenessProbe.httpGet.path }}
|
||||
port: {{ .Values.master.port }}
|
||||
scheme: {{ .Values.allInOne.livenessProbe.httpGet.scheme }}
|
||||
scheme: {{ .Values.allInOne.livenessProbe.scheme }}
|
||||
initialDelaySeconds: {{ .Values.allInOne.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.allInOne.livenessProbe.periodSeconds }}
|
||||
successThreshold: {{ .Values.allInOne.livenessProbe.successThreshold }}
|
||||
@@ -420,31 +374,26 @@ spec:
|
||||
path: {{ .Values.allInOne.data.hostPathPrefix }}/seaweedfs-all-in-one-data/
|
||||
type: DirectoryOrCreate
|
||||
{{- else if eq .Values.allInOne.data.type "persistentVolumeClaim" }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "seaweedfs.name" . }}-all-in-one-data
|
||||
{{- else if eq .Values.allInOne.data.type "existingClaim" }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.allInOne.data.claimName }}
|
||||
{{- else if eq .Values.allInOne.data.type "emptyDir" }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if and .Values.allInOne.s3.enabled (or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth) }}
|
||||
{{- if and .Values.allInOne.s3.enabled (or .Values.s3.enableAuth .Values.filer.s3.enableAuth) }}
|
||||
- name: config-s3-users
|
||||
secret:
|
||||
defaultMode: 420
|
||||
secretName: {{ default (printf "%s-s3-secret" (include "seaweedfs.name" .)) (or .Values.allInOne.s3.existingConfigSecret .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret) }}
|
||||
secretName: {{ default (printf "%s-s3-secret" (include "seaweedfs.name" .)) (or .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret) }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sftp.enabled }}
|
||||
- name: config-ssh
|
||||
secret:
|
||||
defaultMode: 420
|
||||
secretName: {{ default (printf "%s-sftp-ssh-secret" (include "seaweedfs.name" .)) (or .Values.allInOne.sftp.existingSshConfigSecret .Values.sftp.existingSshConfigSecret) }}
|
||||
{{- if or .Values.allInOne.sftp.enableAuth .Values.sftp.enableAuth }}
|
||||
secretName: {{ default (printf "%s-sftp-ssh-secret" (include "seaweedfs.name" .)) .Values.sftp.existingSshConfigSecret }}
|
||||
- name: config-users
|
||||
secret:
|
||||
defaultMode: 420
|
||||
secretName: {{ default (printf "%s-sftp-secret" (include "seaweedfs.name" .)) (or .Values.allInOne.sftp.existingConfigSecret .Values.sftp.existingConfigSecret) }}
|
||||
{{- end }}
|
||||
secretName: {{ default (printf "%s-sftp-secret" (include "seaweedfs.name" .)) .Values.sftp.existingConfigSecret }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.notificationConfig }}
|
||||
- name: notification-config
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user