Compare commits

..
17 Commits
Author SHA1 Message Date
chrislu 2f86f10fe1 Merge branch 'master' into message_send 2022-09-25 23:38:27 -07:00
chrislu 264c138c1d minor 2022-09-25 19:00:34 -07:00
chrislu 24317a9763 async wait 2022-09-25 17:57:26 -07:00
chrislu 3da8d7309a Merge branch 'master' into message_send 2022-09-25 17:46:12 -07:00
chrislu 648b33beb0 add future list 2022-09-25 17:42:23 -07:00
chrislu 1484cb224f Merge branch 'master' into message_send 2022-09-25 13:50:02 -07:00
chrislu 600d2f92a4 add message pipeline 2022-09-25 11:19:05 -07:00
chrislu 15b6c7a6f0 remove file 2022-09-22 23:21:47 -07:00
chrislu bfd0601fd3 refactoring 2022-09-22 23:21:13 -07:00
chrislu 977c8e7830 add todo 2022-09-18 16:49:10 -07:00
chrislu e264fa6e28 volume server: rename readBufferSize to readBufferSizeMB 2022-09-17 10:50:06 -07:00
chrislu dcbded20bb minor 2022-09-17 10:24:47 -07:00
chrislu 1c79301f89 Merge branch 'master' into message_send 2022-09-17 10:18:14 -07:00
chrislu 956ce6416f Merge branch 'master' into message_send 2022-09-14 00:07:27 -07:00
chrislu 3f4400dc03 Merge branch 'master' into message_send 2022-09-05 10:47:13 -07:00
chrislu daa086fd72 segment infos 2022-09-04 18:14:46 -07:00
chrislu 7e33915a26 qsend to broker 2022-09-04 01:29:01 -07:00
3160 changed files with 45348 additions and 844038 deletions
+1 -5
View File
@@ -5,6 +5,7 @@
# How are we solving the problem?
# How is the PR tested?
@@ -12,8 +13,3 @@
# Checks
- [ ] I have added unit tests if possible.
- [ ] I will add related wiki document changes and link to this PR after merging.
- [ ] All AI code review comments have been addressed. No more comments to fix if reviewed again. Reviewer may request additional gemini and copilot reviews.
# Checks for AI generated PRs
- [ ] I have reviewed every line of code.
- [ ] The PR is kept as minimum as possible. Large PRs would not be accepted.
+10 -15
View File
@@ -4,10 +4,6 @@ on:
push:
branches: [ master ]
concurrency:
group: binaries-dev-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
@@ -22,7 +18,6 @@ jobs:
- name: Delete old release assets
uses: mknejp/delete-release-assets@v1
continue-on-error: true
with:
token: ${{ github.token }}
tag: dev
@@ -43,13 +38,13 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -58,14 +53,14 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-large-disk
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -73,7 +68,7 @@ jobs:
release_tag: dev
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-normal-disk
@@ -92,13 +87,13 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -107,14 +102,14 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-large-disk
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -122,7 +117,7 @@ jobs:
release_tag: dev
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed-normal-disk
+5 -5
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -38,13 +38,13 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -52,7 +52,7 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
+5 -5
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -38,13 +38,13 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -52,7 +52,7 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
+5 -5
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -38,13 +38,13 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -52,7 +52,7 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
+5 -5
View File
@@ -28,9 +28,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -38,13 +38,13 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -52,7 +52,7 @@ jobs:
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
+7 -7
View File
@@ -28,32 +28,32 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
build_flags: -tags elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
build_flags: -tags elastic,ydb,gocdk,tikv
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,tikv
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
-59
View File
@@ -1,59 +0,0 @@
# This is a basic workflow to help you get started with Actions
name: "go: build versioned binaries for openbsd"
on:
push:
tags:
- '*'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_openbsd:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:
goos: [openbsd]
goarch: [amd64, arm, arm64]
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is
ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}}
# Where to run `go build .`
project_path: weed
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
+4 -4
View File
@@ -18,11 +18,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
uses: github/codeql-action/init@v2
# Override language selection by uncommenting this and choosing your languages
with:
languages: go
@@ -30,7 +30,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below).
- name: Autobuild
uses: github/codeql-action/autobuild@v4
uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -44,4 +44,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
uses: github/codeql-action/analyze@v2
+22 -108
View File
@@ -3,102 +3,24 @@ name: "docker: build dev containers"
on:
push:
branches: [ master ]
workflow_dispatch: {}
permissions:
contents: read
jobs:
# ── Pre-build Rust volume server binaries natively ──────────────────
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-dev-${{ matrix.target }}-
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: weed-volume-normal-${{ matrix.arch }}
build-dev-containers:
needs: [build-rust-binaries]
runs-on: [ubuntu-latest]
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Download pre-built Rust binaries
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-normal-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Docker meta
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
@@ -109,40 +31,32 @@ jobs:
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.1.0
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.1.0
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build
uses: docker/build-push-action@v7
-
name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
@@ -1,169 +0,0 @@
name: "docker: build foundationdb image by version"
on:
pull_request:
branches: [ master, main ]
paths:
- 'weed/filer/foundationdb/**'
- 'test/foundationdb/**'
- 'docker/Dockerfile.foundationdb_large'
- 'docker/filer_foundationdb.toml'
- '.github/workflows/container_foundationdb_version.yml'
workflow_dispatch:
inputs:
fdb_version:
description: 'FoundationDB version to build (e.g. 7.4.5)'
required: true
default: '7.4.5'
seaweedfs_ref:
description: 'SeaweedFS git tag, branch, or commit to build'
required: true
default: 'master'
image_tag:
description: 'Optional Docker tag suffix (defaults to foundationdb_<fdb>_seaweedfs_<ref>)'
required: false
default: ''
permissions:
contents: read
jobs:
build-foundationdb-image:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install FoundationDB client libraries
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y ca-certificates wget
FDB_VERSION="${{ inputs.fdb_version || '7.4.5' }}"
case "${FDB_VERSION}_amd64" in
"7.4.5_amd64") EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;;
"7.3.43_amd64") EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;;
*)
echo "Unsupported FoundationDB version ${FDB_VERSION} for CI client install" >&2
exit 1 ;;
esac
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
wget --timeout=30 --tries=3 -O "${PACKAGE}" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}"
echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c -
sudo dpkg -i "${PACKAGE}"
rm "${PACKAGE}"
sudo ldconfig
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: go.mod
- name: Run FoundationDB tagged tests
env:
CGO_ENABLED: 1
run: |
go test ./weed/filer/foundationdb -tags foundationdb -count=1
- name: Prepare Docker tag
id: tag
env:
FDB_VERSION_INPUT: ${{ inputs.fdb_version }}
SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
EVENT_NAME: ${{ github.event_name }}
HEAD_REF: ${{ github.head_ref }}
REF_NAME: ${{ github.ref_name }}
run: |
set -euo pipefail
sanitize() {
local value="$1"
value="${value,,}"
value="${value// /-}"
value="${value//[^a-z0-9_.-]/-}"
value="${value#-}"
value="${value%-}"
printf '%s' "$value"
}
version="${FDB_VERSION_INPUT}"
seaweed="${SEAWEEDFS_REF_INPUT}"
tag="${CUSTOM_TAG_INPUT}"
# Use defaults for PR builds
if [ -z "$version" ]; then
version="7.4.5"
fi
if [ -z "$seaweed" ]; then
if [ "$EVENT_NAME" = "pull_request" ]; then
seaweed="${HEAD_REF}"
else
seaweed="${REF_NAME}"
fi
fi
sanitized_version="$(sanitize "$version")"
if [ -z "$sanitized_version" ]; then
echo "Unable to sanitize FoundationDB version '$version'." >&2
exit 1
fi
sanitized_seaweed="$(sanitize "$seaweed")"
if [ -z "$sanitized_seaweed" ]; then
echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
exit 1
fi
if [ -z "$tag" ]; then
tag="foundationdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
else
tag="$(sanitize "$tag")"
fi
if [ -z "$tag" ]; then
echo "Resulting Docker tag is empty." >&2
exit 1
fi
echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Determine branch to build
id: branch
run: |
if [ -n "${{ inputs.seaweedfs_ref }}" ]; then
echo "branch=${{ inputs.seaweedfs_ref }}" >> "$GITHUB_OUTPUT"
elif [ "${{ github.event_name }}" = "pull_request" ]; then
echo "branch=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
else
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push image
uses: docker/build-push-action@v7
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.foundationdb_large
build-args: |
FDB_VERSION=${{ inputs.fdb_version || '7.4.5' }}
BRANCH=${{ steps.branch.outputs.branch }}
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
# Currently only building for amd64. To enable ARM64, verify package availability and add checksums.
platforms: linux/amd64
tags: ${{ steps.tag.outputs.full_image || 'seaweedfs:foundationdb-test' }}
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
+31 -472
View File
@@ -4,505 +4,64 @@ on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
source_ref:
description: 'Git ref to build (branch, tag, or commit SHA)'
required: true
default: 'master'
image_tag:
description: 'Docker tag to publish (without variant suffix)'
required: true
default: 'latest'
variant:
description: 'Variant to build manually'
required: true
type: choice
default: all
options:
- all
- standard
- large_disk
publish:
description: 'Publish images and manifests'
required: true
type: boolean
default: false
workflow_dispatch: {}
permissions:
contents: read
security-events: write
jobs:
setup:
runs-on: ubuntu-latest
outputs:
variants: ${{ steps.set-variants.outputs.variants }}
publish: ${{ steps.set-publish.outputs.publish }}
steps:
- name: Select variants for this run
id: set-variants
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.variant }}" != "all" ]; then
variants="[\"${{ github.event.inputs.variant }}\"]"
else
variants='["standard","large_disk"]'
fi
echo "variants=$variants" >> "$GITHUB_OUTPUT"
- name: Select publish mode
id: set-publish
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
else
echo "publish=true" >> "$GITHUB_OUTPUT"
fi
# ── Pre-build Rust volume server binaries natively ──────────────────
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-${{ matrix.target }}-
- name: Build large-disk variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: |
weed-volume-large-disk-${{ matrix.arch }}
weed-volume-normal-${{ matrix.arch }}
build:
needs: [setup, build-rust-binaries]
runs-on: ubuntu-latest
strategy:
matrix:
platform: [amd64, arm64, arm, 386]
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
build-latest-container:
runs-on: [ubuntu-latest]
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
df -h
# Remove pre-installed tools
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Clean package managers
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
# Clean Docker aggressively
sudo docker system prune -af --volumes
# Clean Go cache if it exists
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Configure variant
id: config
run: |
if [ "${{ matrix.variant }}" == "large_disk" ]; then
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
else
echo "tag_suffix=" >> $GITHUB_OUTPUT
echo "build_args=" >> $GITHUB_OUTPUT
echo "rust_variant=normal" >> $GITHUB_OUTPUT
fi
- name: Download pre-built Rust binaries
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Docker meta
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
tags: |
type=raw,value=latest
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: matrix.platform != 'amd64'
uses: docker/setup-qemu-action@v4
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.1.0
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.1.0
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
uses: docker/build-push-action@v7
env:
DOCKER_BUILDKIT: 1
-
name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ needs.setup.outputs.publish == 'true' }}
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/${{ matrix.platform }}
# Push to GHCR only during build to avoid Docker Hub rate limits
tags: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.platform }}
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.platform }}
build-args: |
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
${{ steps.config.outputs.build_args }}
- name: Clean up build artifacts
if: always()
run: |
# Clean up Docker build cache and temporary files
sudo docker system prune -f
# Remove Go build cache
sudo rm -rf /tmp/go-build*
trivy-scan:
runs-on: ubuntu-latest
needs: [setup, build, build-rust-binaries]
strategy:
matrix:
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
steps:
- name: Configure variant
id: config
run: |
if [ "${{ matrix.variant }}" == "large_disk" ]; then
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
else
echo "tag_suffix=" >> $GITHUB_OUTPUT
fi
- name: Login to GHCR
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.1.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Checkout for local scan build
if: needs.setup.outputs.publish != 'true'
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Download pre-built Rust binaries for local scan
if: needs.setup.outputs.publish != 'true'
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context for local scan
if: needs.setup.outputs.publish != 'true'
run: |
rust_variant="normal"
if [ "${{ matrix.variant }}" == "large_disk" ]; then
rust_variant="large-disk"
fi
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Create BuildKit config for local scan build
if: needs.setup.outputs.publish != 'true'
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx for local scan build
if: needs.setup.outputs.publish != 'true'
uses: docker/setup-buildx-action@v4
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Build local scan image tarball
if: needs.setup.outputs.publish != 'true'
uses: docker/build-push-action@v7
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
file: ./docker/Dockerfile.go_build
platforms: linux/amd64
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
build-args: |
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
- name: Trivy report (published image)
if: needs.setup.outputs.publish == 'true'
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: image
# Scan amd64 only - OS packages are identical across architectures
# since they all use the same alpine base, so a single-arch scan
# provides sufficient coverage without multiplying CI time.
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
- name: Trivy report (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
- name: Upload Trivy scan results to GitHub Security
uses: github/codeql-action/upload-sarif@v4
if: always()
with:
sarif_file: trivy-results.sarif
- name: Trivy gate (published image)
if: needs.setup.outputs.publish == 'true'
# Gate only on fixable high/critical vulnerabilities. Non-fixable
# findings are still visible in the SARIF upload above.
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: image
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
format: table
exit-code: '1'
skip-setup-trivy: true
- name: Trivy gate (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
format: table
exit-code: '1'
skip-setup-trivy: true
create-manifest:
runs-on: ubuntu-latest
needs: [setup, build, trivy-scan]
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
strategy:
matrix:
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Configure variant
id: config
run: |
if [ "${{ matrix.variant }}" == "large_disk" ]; then
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
else
echo "tag_suffix=" >> $GITHUB_OUTPUT
fi
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
- name: Login to Docker Hub
uses: docker/login-action@v4.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4.1.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
run: |
# Install crane for efficient multi-arch image copying
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Create and push manifest
run: |
SUFFIX="${{ steps.config.outputs.tag_suffix }}"
BASE_TAG="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}"
# Create manifest on GHCR first (no rate limits)
echo "Creating GHCR manifest (no rate limits)..."
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
# Copy the complete multi-arch image from GHCR to Docker Hub
# This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
echo "Copying manifest from GHCR to Docker Hub..."
# Function to retry with exponential backoff for Docker Hub operations
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Use crane or skopeo to copy, fallback to docker if not available
if command -v crane &> /dev/null; then
echo "Using crane to copy..."
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
elif command -v skopeo &> /dev/null; then
echo "Using skopeo to copy..."
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
else
echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
# Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
fi
+57
View File
@@ -0,0 +1,57 @@
name: "docker: build release containers for normal volume"
on:
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
+59
View File
@@ -0,0 +1,59 @@
name: "docker: build release containers for large volume"
on:
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build-large-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: TAGS=5BytesOffset
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
+58
View File
@@ -0,0 +1,58 @@
name: "docker: build release containers for rocksdb"
on:
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build-large-release-container_rocksdb:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk_rocksdb
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.rocksdb_large
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
+58
View File
@@ -0,0 +1,58 @@
name: "docker: build release containers for all tags"
on:
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_full
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: TAGS=elastic,ydb,gocdk,tikv
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
+58
View File
@@ -0,0 +1,58 @@
name: "docker: build release containers for all tags and large volume"
on:
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk_full
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
@@ -1,79 +0,0 @@
name: "docker: build release containers for foundationdb"
on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag to publish (e.g. 3.93)'
required: true
default: ''
permissions:
contents: read
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
jobs:
build-large-release-container_foundationdb:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@v6
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
with:
images: |
chrislusf/seaweedfs
tags: |
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@v4
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Determine branch to build
id: branch
run: |
if [ "${{ github.event_name }}" = "push" ] && [ -n "${{ github.ref_name }}" ]; then
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
else
echo "branch=master" >> "$GITHUB_OUTPUT"
fi
-
name: Build
uses: docker/build-push-action@v7
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.foundationdb_large
build-args: |
BRANCH=${{ steps.branch.outputs.branch }}
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
@@ -1,378 +0,0 @@
name: "docker: build all release containers (unified)"
on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
variant:
description: 'Variant to build manually'
required: true
type: choice
default: all
options:
- all
- normal
- large_disk
- full
- large_disk_full
- rocksdb
release_tag:
description: 'Release tag to publish (e.g. 3.93)'
required: true
default: ''
rocksdb_version:
description: 'RocksDB git tag to use when variant=rocksdb'
required: false
default: 'v10.10.1'
permissions:
contents: read
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
# Limit concurrent builds to avoid rate limits
concurrency:
group: release-${{ github.ref }}
cancel-in-progress: false
jobs:
# ── Pre-build Rust volume server binaries natively ──────────────────
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
# emulated cargo build into ~15 minutes of native compilation.
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-${{ matrix.target }}-
- name: Build large-disk variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: |
weed-volume-large-disk-${{ matrix.arch }}
weed-volume-normal-${{ matrix.arch }}
# ── Build Docker containers ─────────────────────────────────────────
build:
needs: [build-rust-binaries]
runs-on: ubuntu-latest
strategy:
# Build sequentially to avoid rate limits
max-parallel: 2
matrix:
include:
# Normal volume - multi-arch
- variant: normal
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
dockerfile: ./docker/Dockerfile.go_build
build_args: ""
tag_suffix: ""
rust_variant: normal
# Large disk - multi-arch
- variant: large_disk
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset
tag_suffix: _large_disk
rust_variant: large-disk
# Full tags - multi-arch
- variant: full
platforms: linux/amd64,linux/arm64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _full
rust_variant: normal
# Large disk + full tags - multi-arch
- variant: large_disk_full
platforms: linux/amd64,linux/arm64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _large_disk_full
rust_variant: large-disk
# RocksDB large disk - amd64 only
- variant: rocksdb
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.rocksdb_large
build_args: ""
tag_suffix: _large_disk_rocksdb
rust_variant: large-disk
steps:
- name: Checkout
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/checkout@v6
- name: Download pre-built Rust binaries
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Free Disk Space
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
echo "Available disk space before cleanup:"
df -h
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
sudo docker system prune -af --volumes
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
id: docker_meta
uses: docker/metadata-action@v6
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
flavor: latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
uses: docker/setup-qemu-action@v4
- name: Create BuildKit config
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/setup-buildx-action@v4
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v4.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v4.1.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build and push ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/build-push-action@v7
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platforms }}
# Push to GHCR to avoid Docker Hub rate limits on pulls
tags: |
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha,scope=${{ matrix.variant }}
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
build-args: |
${{ matrix.build_args }}
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
- name: Clean up build artifacts
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
run: |
sudo docker system prune -f
sudo rm -rf /tmp/go-build*
copy-to-dockerhub:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
strategy:
matrix:
variant: [normal, large_disk, full, large_disk_full, rocksdb]
include:
- variant: normal
tag_suffix: ""
- variant: large_disk
tag_suffix: _large_disk
- variant: full
tag_suffix: _full
- variant: large_disk_full
tag_suffix: _large_disk_full
- variant: rocksdb
tag_suffix: _large_disk_rocksdb
steps:
- name: Login to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.1.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
# Function to retry with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Copy multi-arch image from GHCR to Docker Hub with retry
# This is much more efficient than pulling/pushing individual arch images
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
retry_with_backoff crane copy \
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
helm-release:
runs-on: ubuntu-latest
needs: [copy-to-dockerhub]
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
permissions:
contents: write
pages: write
steps:
- uses: actions/checkout@v6
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@v1.7.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
charts_dir: k8s/charts
target_dir: helm
branch: gh-pages
helm_version: "3.18.4"
@@ -1,110 +0,0 @@
name: "docker: build rocksdb image by version"
on:
workflow_dispatch:
inputs:
rocksdb_version:
description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
required: true
default: 'v10.10.1'
seaweedfs_ref:
description: 'SeaweedFS git tag, branch, or commit to build'
required: true
default: 'master'
image_tag:
description: 'Optional Docker tag suffix (defaults to rocksdb_<rocksdb>_seaweedfs_<ref>)'
required: false
default: ''
permissions:
contents: read
jobs:
build-rocksdb-image:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
- name: Prepare Docker tag
id: tag
env:
ROCKSDB_VERSION_INPUT: ${{ inputs.rocksdb_version }}
SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
run: |
set -euo pipefail
sanitize() {
local value="$1"
value="${value,,}"
value="${value// /-}"
value="${value//[^a-z0-9_.-]/-}"
value="${value#-}"
value="${value%-}"
printf '%s' "$value"
}
version="${ROCKSDB_VERSION_INPUT}"
seaweed="${SEAWEEDFS_REF_INPUT}"
tag="${CUSTOM_TAG_INPUT}"
if [ -z "$version" ]; then
echo "RocksDB version input is required." >&2
exit 1
fi
if [ -z "$seaweed" ]; then
echo "SeaweedFS ref input is required." >&2
exit 1
fi
sanitized_version="$(sanitize "$version")"
if [ -z "$sanitized_version" ]; then
echo "Unable to sanitize RocksDB version '$version'." >&2
exit 1
fi
sanitized_seaweed="$(sanitize "$seaweed")"
if [ -z "$sanitized_seaweed" ]; then
echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
exit 1
fi
if [ -z "$tag" ]; then
tag="rocksdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
fi
tag="${tag,,}"
tag="${tag// /-}"
tag="${tag//[^a-z0-9_.-]/-}"
tag="${tag#-}"
tag="${tag%-}"
if [ -z "$tag" ]; then
echo "Resulting Docker tag is empty." >&2
exit 1
fi
echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
- name: Login to Docker Hub
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push image
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
with:
context: ./docker
push: true
file: ./docker/Dockerfile.rocksdb_large
build-args: |
ROCKSDB_VERSION=${{ inputs.rocksdb_version }}
BRANCH=${{ inputs.seaweedfs_ref }}
platforms: linux/amd64
tags: ${{ steps.tag.outputs.full_image }}
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-171
View File
@@ -1,171 +0,0 @@
# This workflow will build and deploy the SeaweedFS telemetry server
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
name: Deploy Telemetry Server
on:
workflow_dispatch:
inputs:
setup:
description: 'Run first-time server setup'
required: true
type: boolean
default: false
deploy:
description: 'Deploy telemetry server to remote server'
required: true
type: boolean
default: false
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build Telemetry Server
if: github.event_name == 'workflow_dispatch' && inputs.deploy
run: |
go mod tidy
echo "Building telemetry server..."
GOOS=linux GOARCH=amd64 go build -o telemetry-server ./telemetry/server/main.go
ls -la telemetry-server
echo "Build completed successfully"
- name: First-time Server Setup
if: github.event_name == 'workflow_dispatch' && inputs.setup
env:
SSH_PRIVATE_KEY: ${{ secrets.TELEMETRY_SSH_PRIVATE_KEY }}
REMOTE_HOST: ${{ secrets.TELEMETRY_HOST }}
REMOTE_USER: ${{ secrets.TELEMETRY_USER }}
run: |
mkdir -p ~/.ssh
echo "$SSH_PRIVATE_KEY" > ~/.ssh/deploy_key
chmod 600 ~/.ssh/deploy_key
echo "Host *" > ~/.ssh/config
echo " StrictHostKeyChecking no" >> ~/.ssh/config
# Create all required directories with proper permissions
ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST "
mkdir -p ~/seaweedfs-telemetry/bin ~/seaweedfs-telemetry/logs ~/seaweedfs-telemetry/data ~/seaweedfs-telemetry/tmp && \
chmod 755 ~/seaweedfs-telemetry/logs && \
chmod 755 ~/seaweedfs-telemetry/data && \
touch ~/seaweedfs-telemetry/logs/telemetry.log ~/seaweedfs-telemetry/logs/telemetry.error.log && \
chmod 644 ~/seaweedfs-telemetry/logs/*.log"
# Create systemd service file
echo "
[Unit]
Description=SeaweedFS Telemetry Server
After=network.target
[Service]
Type=simple
User=$REMOTE_USER
WorkingDirectory=/home/$REMOTE_USER/seaweedfs-telemetry
ExecStart=/home/$REMOTE_USER/seaweedfs-telemetry/bin/telemetry-server -port=8353
Restart=always
RestartSec=5
StandardOutput=append:/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.log
StandardError=append:/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.error.log
[Install]
WantedBy=multi-user.target" > telemetry.service
# Setup logrotate configuration
echo "# SeaweedFS Telemetry service log rotation
/home/$REMOTE_USER/seaweedfs-telemetry/logs/*.log {
daily
rotate 30
compress
delaycompress
missingok
notifempty
create 644 $REMOTE_USER $REMOTE_USER
postrotate
systemctl restart telemetry.service
endscript
}" > telemetry_logrotate
# Copy configuration files
scp -i ~/.ssh/deploy_key telemetry/grafana-dashboard.json $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
scp -i ~/.ssh/deploy_key telemetry/prometheus.yml $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
# Copy and install service and logrotate files
scp -i ~/.ssh/deploy_key telemetry.service telemetry_logrotate $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST "
sudo mv ~/seaweedfs-telemetry/telemetry.service /etc/systemd/system/ && \
sudo mv ~/seaweedfs-telemetry/telemetry_logrotate /etc/logrotate.d/seaweedfs-telemetry && \
sudo systemctl daemon-reload && \
sudo systemctl enable telemetry.service"
echo "✅ First-time setup completed successfully!"
echo "📋 Next step: Run the deployment to install the telemetry server binary"
echo " 1. Go to GitHub Actions → Deploy Telemetry Server"
echo " 2. Click 'Run workflow'"
echo " 3. Check 'Deploy telemetry server to remote server'"
echo " 4. Click 'Run workflow'"
rm -f ~/.ssh/deploy_key
- name: Deploy Telemetry Server to Remote Server
if: github.event_name == 'workflow_dispatch' && inputs.deploy
env:
SSH_PRIVATE_KEY: ${{ secrets.TELEMETRY_SSH_PRIVATE_KEY }}
REMOTE_HOST: ${{ secrets.TELEMETRY_HOST }}
REMOTE_USER: ${{ secrets.TELEMETRY_USER }}
run: |
mkdir -p ~/.ssh
echo "$SSH_PRIVATE_KEY" > ~/.ssh/deploy_key
chmod 600 ~/.ssh/deploy_key
echo "Host *" > ~/.ssh/config
echo " StrictHostKeyChecking no" >> ~/.ssh/config
# Create temp directory and copy binary
ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST "mkdir -p ~/seaweedfs-telemetry/tmp"
scp -i ~/.ssh/deploy_key telemetry-server $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/tmp/
# Copy updated configuration files
scp -i ~/.ssh/deploy_key telemetry/grafana-dashboard.json $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
scp -i ~/.ssh/deploy_key telemetry/prometheus.yml $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
# Check if service exists and deploy accordingly
ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST "
if systemctl list-unit-files telemetry.service >/dev/null 2>&1; then
echo 'Service exists, performing update...'
sudo systemctl stop telemetry.service
mkdir -p ~/seaweedfs-telemetry/bin
mv ~/seaweedfs-telemetry/tmp/telemetry-server ~/seaweedfs-telemetry/bin/
chmod +x ~/seaweedfs-telemetry/bin/telemetry-server
sudo systemctl start telemetry.service
sudo systemctl status telemetry.service
else
echo 'ERROR: telemetry.service not found!'
echo 'Please run the first-time setup before deploying.'
echo 'Go to GitHub Actions → Deploy Telemetry Server → Run workflow → Check \"Run first-time server setup\"'
exit 1
fi"
# Verify deployment
ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST "
echo 'Waiting for service to start...'
sleep 5
curl -f http://localhost:8353/health || echo 'Health check failed'"
rm -f ~/.ssh/deploy_key
- name: Notify Deployment Status
if: always()
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "✅ Telemetry server deployment successful"
echo "Dashboard: http://${{ secrets.TELEMETRY_HOST }}:8353"
echo "Metrics: http://${{ secrets.TELEMETRY_HOST }}:8353/metrics"
else
echo "❌ Telemetry server deployment failed"
fi
+2 -2
View File
@@ -9,6 +9,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
- name: 'Dependency Review'
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
uses: actions/dependency-review-action@23d1ffffb6fa5401173051ec21eba8c35242733f
+27 -90
View File
@@ -21,112 +21,49 @@ jobs:
e2e:
name: FUSE Mount
runs-on: ubuntu-22.04
timeout-minutes: 30
timeout-minutes: 15
steps:
- name: Set up Go 1.x
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
with:
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Cache Docker layers
uses: actions/cache@v5
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-e2e-
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Install dependencies
run: |
# Use faster mirrors and install with timeout
sudo rm -f /etc/apt/sources.list.d/azure-cli.list /etc/apt/sources.list.d/microsoft-prod.list
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse" | sudo tee /etc/apt/sources.list
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
sudo apt-get update --fix-missing
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends fuse
# Verify FUSE installation
echo "FUSE version: $(fusermount --version 2>&1 || echo 'fusermount not found')"
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
sudo apt-get update
sudo apt-get install -y fuse
- name: Start SeaweedFS
timeout-minutes: 15
run: |
# Enable Docker buildkit for better caching
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
timeout-minutes: 5
run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
# Build with retry logic
for i in {1..3}; do
echo "Build attempt $i/3"
if make build_e2e; then
echo "Build successful on attempt $i"
break
elif [ $i -eq 3 ]; then
echo "Build failed after 3 attempts"
exit 1
else
echo "Build attempt $i failed, retrying in 30 seconds..."
sleep 30
fi
done
# Start services with wait
docker compose -f ./compose/e2e-mount.yml up --wait
- name: Rotate buildx cache
if: always()
run: |
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
rm -rf /tmp/.buildx-cache
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
- name: Run FIO 4k
timeout-minutes: 15
- name: Run FIO
timeout-minutes: 5
run: |
echo "Starting FIO at: $(date)"
# Concurrent r/w
echo 'Run randrw with size=16M bs=4k'
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --group_reporting --runtime=30 --time_based=1
echo "Verify FIO at: $(date)"
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
echo 'Run randrw with size=16M bs=128k'
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
echo 'Run randrw with size=16M bs=1m'
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
# Verified write
echo 'Run randwrite with size=16M bs=4k'
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
- name: Run FIO 128k
timeout-minutes: 15
run: |
echo "Starting FIO at: $(date)"
# Concurrent r/w
echo 'Run randrw with size=16M bs=128k'
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
echo "Verify FIO at: $(date)"
# Verified write
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
echo 'Run randwrite with size=16M bs=128k'
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
- name: Run FIO 1MB
timeout-minutes: 15
run: |
echo "Starting FIO at: $(date)"
# Concurrent r/w
echo 'Run randrw with size=16M bs=1m'
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
echo "Verify FIO at: $(date)"
# Verified write
echo 'Run randwrite with size=16M bs=1m'
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
- name: Save logs
if: always()
@@ -142,7 +79,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v3
with:
name: output-logs
path: docker/output.log
@@ -1,61 +0,0 @@
name: "EC Integration Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
permissions:
contents: read
jobs:
ec-integration-tests:
name: EC Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run EC Integration Tests
working-directory: test/erasure_coding
run: |
go test -v
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting server logs from temp directories..."
mkdir -p /tmp/ec-test-logs
# Find all temp directories created by the tests (they persist on failure with t.TempDir())
find /tmp -maxdepth 1 -type d \( -name "TestEC*" -o -name "TestDisk*" -o -name "TestCross*" -o -name "TestEvacuation*" \) 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
# Copy the entire directory structure to preserve organization
cp -r "$dir" /tmp/ec-test-logs/ 2>/dev/null || true
fi
done
# List what we collected
echo "Collected logs:"
find /tmp/ec-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: ec-integration-test-logs
path: |
/tmp/ec-test-logs/
test/erasure_coding/
if-no-files-found: warn
-49
View File
@@ -1,49 +0,0 @@
name: EC Integration Tests
on:
push:
branches: [ master ]
paths:
- 'weed/admin/**'
- 'weed/worker/**'
- 'test/erasure_coding/admin_dockertest/**'
- '.github/workflows/ec-integration.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/admin/**'
- 'weed/worker/**'
- 'test/erasure_coding/admin_dockertest/**'
- '.github/workflows/ec-integration.yml'
jobs:
ec-integration-test:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build weed binary
run: |
cd weed
go build -o ../weed_bin
- name: Run EC integration tests
run: |
cd test/erasure_coding/admin_dockertest
go test -v -timeout 15m ec_integration_test.go
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: ec-test-logs
path: test/erasure_coding/admin_dockertest/tmp/logs/
retention-days: 7
@@ -1,63 +0,0 @@
name: "FUSE DLM Integration Tests"
on:
pull_request:
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/cluster/**'
- 'test/fuse_dlm/**'
- '.github/workflows/fuse-dlm-integration.yml'
push:
branches: [master]
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/cluster/**'
- 'test/fuse_dlm/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-dlm-integration:
name: FUSE DLM Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install FUSE dependencies
run: |
sudo apt-get update
sudo apt-get install -y libfuse3-dev
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
- name: Build SeaweedFS
run: go build -o weed/weed -buildvcs=false ./weed
- name: Run DLM integration tests
timeout-minutes: 25
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: fuse-dlm-test-logs
path: /tmp/seaweedfs-fuse-dlm-logs/
retention-days: 3
-81
View File
@@ -1,81 +0,0 @@
name: "FUSE Integration Tests"
on:
push:
branches: [ master, main ]
paths:
- 'weed/**'
- 'test/fuse_integration/**'
- '.github/workflows/fuse-integration.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/**'
- 'test/fuse_integration/**'
- '.github/workflows/fuse-integration.yml'
concurrency:
group: ${{ github.head_ref }}/fuse-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-integration:
name: FUSE Integration Testing
runs-on: ubuntu-22.04
timeout-minutes: 50
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install FUSE and dependencies
run: |
sudo apt-get update
# fuse3 is pre-installed on ubuntu-22.04 runners and conflicts
# with the legacy fuse package, so only install the dev headers.
sudo apt-get install -y libfuse3-dev
# Allow non-root FUSE mounts with allow_other
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
# Verify FUSE installation
fusermount3 --version || fusermount --version || true
ls -la /dev/fuse
- name: Build SeaweedFS
run: |
cd weed
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -o weed .
chmod +x weed
./weed version
# Make weed binary available in PATH for the test framework
sudo cp weed /usr/local/bin/weed
- name: Install test dependencies
run: |
cd test/fuse_integration
go mod download
- name: Run FUSE Integration Tests
run: |
set -o pipefail
cd test/fuse_integration
echo "Running full FUSE integration test suite..."
go test -v -count=1 -timeout=45m ./... 2>&1 | tee /tmp/fuse-test-output.log
- name: Upload Test Logs
if: always()
uses: actions/upload-artifact@v7
with:
name: fuse-integration-test-results
path: |
/tmp/fuse-test-output.log
/tmp/seaweedfs-fuse-logs/
retention-days: 7
@@ -1,69 +0,0 @@
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
on:
pull_request:
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/filer/mount_peer_registry*.go'
- 'weed/server/filer_grpc_server_mount_peer.go'
- 'weed/pb/mount_peer.proto'
- 'weed/pb/filer.proto'
- 'test/fuse_p2p/**'
- '.github/workflows/fuse-p2p-integration.yml'
push:
branches: [master]
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/filer/mount_peer_registry*.go'
- 'weed/server/filer_grpc_server_mount_peer.go'
- 'weed/pb/mount_peer.proto'
- 'weed/pb/filer.proto'
- 'test/fuse_p2p/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-p2p-integration:
name: FUSE P2P Peer Chunk Sharing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install FUSE dependencies
run: |
sudo apt-get update
sudo apt-get install -y libfuse3-dev
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
- name: Build SeaweedFS
run: go build -o weed/weed -buildvcs=false ./weed
- name: Run P2P integration tests
timeout-minutes: 15
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: fuse-p2p-test-logs
path: /tmp/seaweedfs-fuse-p2p-logs/
retention-days: 3
+16 -40
View File
@@ -15,50 +15,26 @@ permissions:
jobs:
vet:
name: Go Vet
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Get dependencies
run: |
cd weed; go get -v -t -d ./...
- name: Go Vet (excluding protobuf lock copying)
run: |
cd weed
# Run go vet and filter out known protobuf MessageState lock copying warnings
# These are expected in generated protobuf code with embedded sync.Mutex and are safe in practice
go vet -v ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-output.txt
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
if grep -q "vet:" vet-output.txt; then exit 1; fi
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
test:
name: Test
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
- name: Set up Go 1.x
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- name: Get dependencies
run: |
cd weed; go get -v -t -d ./...
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv" -v .
- name: Test
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
-326
View File
@@ -1,326 +0,0 @@
name: "helm: lint and test charts"
on:
push:
branches: [ master ]
paths: ['k8s/**']
pull_request:
branches: [ master ]
paths: ['k8s/**']
permissions:
contents: read
jobs:
lint-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
with:
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v5
with:
version: v3.18.4
- uses: actions/setup-python@v6
with:
python-version: '3.10'
check-latest: true
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.8.0
- name: Run chart-testing (list-changed)
id: list-changed
run: |
changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs k8s/charts)
if [[ -n "$changed" ]]; then
echo "::set-output name=changed::true"
fi
- name: Run chart-testing (lint)
run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
- name: Verify template rendering
run: |
set -e
CHART_DIR="k8s/charts/seaweedfs"
echo "=== Testing default configuration ==="
helm template test $CHART_DIR > /tmp/default.yaml
echo "✓ Default configuration renders successfully"
echo "=== Testing with S3 enabled ==="
helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
echo "✓ S3 deployment renders correctly"
echo "=== Testing with all-in-one mode ==="
helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
echo "✓ All-in-one deployment renders correctly"
echo "=== Testing with security enabled ==="
helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true > /tmp/security.yaml
grep -q "security-config" /tmp/security.yaml
echo "✓ Security configuration renders correctly"
echo "=== Testing with monitoring enabled ==="
helm template test $CHART_DIR \
--set global.seaweedfs.monitoring.enabled=true \
--set global.seaweedfs.monitoring.gatewayHost=prometheus \
--set global.seaweedfs.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
echo "✓ Monitoring configuration renders correctly"
echo "=== Testing with PVC storage ==="
helm template test $CHART_DIR \
--set master.data.type=persistentVolumeClaim \
--set master.data.size=10Gi \
--set master.data.storageClass=standard > /tmp/pvc.yaml
grep -q "PersistentVolumeClaim" /tmp/pvc.yaml
echo "✓ PVC configuration renders correctly"
echo "=== Testing with custom replicas ==="
helm template test $CHART_DIR \
--set master.replicas=3 \
--set filer.replicas=2 \
--set volume.replicas=3 > /tmp/replicas.yaml
echo "✓ Custom replicas configuration renders correctly"
echo "=== Testing filer with S3 gateway ==="
helm template test $CHART_DIR \
--set filer.s3.enabled=true \
--set filer.s3.enableAuth=true > /tmp/filer-s3.yaml
echo "✓ Filer S3 gateway renders correctly"
echo "=== Testing SFTP enabled ==="
helm template test $CHART_DIR --set sftp.enabled=true > /tmp/sftp.yaml
grep -q "seaweedfs-sftp" /tmp/sftp.yaml
echo "✓ SFTP deployment renders correctly"
echo "=== Testing ingress configurations ==="
helm template test $CHART_DIR \
--set master.ingress.enabled=true \
--set filer.ingress.enabled=true \
--set s3.enabled=true \
--set s3.ingress.enabled=true > /tmp/ingress.yaml
grep -q "kind: Ingress" /tmp/ingress.yaml
echo "✓ Ingress configurations render correctly"
echo "=== Testing COSI driver ==="
helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
grep -q "seaweedfs-cosi" /tmp/cosi.yaml
echo "✓ COSI driver renders correctly"
echo ""
echo "=== Testing long release name: service names match DNS references ==="
# Use a release name that, combined with chart name "seaweedfs", exceeds 63 chars.
# fullname = "my-very-long-release-name-that-will-cause-truncation-seaweedfs" (65 chars before trunc)
LONG_RELEASE="my-very-long-release-name-that-will-cause-truncation"
# --- Normal mode: master + filer-client services vs helper-produced addresses ---
helm template "$LONG_RELEASE" $CHART_DIR \
--set s3.enabled=true \
--set global.seaweedfs.createBuckets[0].name=test > /tmp/longname.yaml
# Extract Service names from metadata
MASTER_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \
| grep -- '-master$')
FILER_CLIENT_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \
| grep -- '-filer-client$')
# Extract the hostname from WEED_CLUSTER_SW_MASTER in post-install-bucket-hook
MASTER_ADDR=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname.yaml \
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/')
FILER_ADDR=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname.yaml \
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/')
# Extract the hostname from S3 deployment -filer= argument
S3_FILER_HOST=$(grep '\-filer=' /tmp/longname.yaml \
| head -1 | sed 's/.*-filer=\([^:]*\).*/\1/')
# The address helpers produce "<svc>.<namespace>:<port>"; extract just the svc name
MASTER_ADDR_SVC=$(echo "$MASTER_ADDR" | cut -d. -f1)
FILER_ADDR_SVC=$(echo "$FILER_ADDR" | cut -d. -f1)
S3_FILER_SVC=$(echo "$S3_FILER_HOST" | cut -d. -f1)
echo " master Service.name: $MASTER_SVC"
echo " cluster.masterAddress svc: $MASTER_ADDR_SVC"
echo " filer-client Service.name: $FILER_CLIENT_SVC"
echo " cluster.filerAddress svc: $FILER_ADDR_SVC"
echo " S3 -filer= svc: $S3_FILER_SVC"
[ "$MASTER_SVC" = "$MASTER_ADDR_SVC" ] || { echo "FAIL: master service name mismatch"; exit 1; }
[ "$FILER_CLIENT_SVC" = "$FILER_ADDR_SVC" ] || { echo "FAIL: filer-client service name mismatch"; exit 1; }
[ "$FILER_CLIENT_SVC" = "$S3_FILER_SVC" ] || { echo "FAIL: S3 -filer= does not match filer-client service"; exit 1; }
echo "✓ Normal mode: service names match DNS references with long release name"
# --- All-in-one mode: all-in-one service vs both helper addresses ---
helm template "$LONG_RELEASE" $CHART_DIR \
--set allInOne.enabled=true \
--set global.seaweedfs.createBuckets[0].name=test > /tmp/longname-aio.yaml
AIO_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname-aio.yaml \
| grep -- '-all-in-one$')
AIO_MASTER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname-aio.yaml \
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1)
AIO_FILER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname-aio.yaml \
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1)
echo " all-in-one Service.name: $AIO_SVC"
echo " cluster.masterAddress svc: $AIO_MASTER_ADDR_SVC"
echo " cluster.filerAddress svc: $AIO_FILER_ADDR_SVC"
[ "$AIO_SVC" = "$AIO_MASTER_ADDR_SVC" ] || { echo "FAIL: all-in-one master address mismatch"; exit 1; }
[ "$AIO_SVC" = "$AIO_FILER_ADDR_SVC" ] || { echo "FAIL: all-in-one filer address mismatch"; exit 1; }
echo "✓ All-in-one mode: service names match DNS references with long release name"
echo ""
echo "=== Testing security+S3: no blank lines in shell command blocks ==="
# Render the three manifests that include seaweedfs.s3.tlsArgs:
# filer-statefulset, s3-deployment, all-in-one-deployment
helm template test $CHART_DIR \
--set global.seaweedfs.enableSecurity=true \
--set filer.s3.enabled=true \
--set s3.enabled=true > /tmp/security-s3.yaml
helm template test $CHART_DIR \
--set global.seaweedfs.enableSecurity=true \
--set allInOne.enabled=true \
--set allInOne.s3.enabled=true > /tmp/security-aio.yaml
pip install pyyaml -q
python3 - /tmp/security-s3.yaml /tmp/security-aio.yaml <<'PYEOF'
import yaml, sys
errors = []
for path in sys.argv[1:]:
with open(path) as f:
docs = list(yaml.safe_load_all(f))
for doc in docs:
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
continue
name = doc["metadata"]["name"]
for c in doc["spec"]["template"]["spec"].get("containers", []):
cmd = c.get("command", [])
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
script = cmd[2]
for i, line in enumerate(script.splitlines(), 1):
if line.strip() == "":
errors.append(f"{path}: {name}/{c['name']} has blank line at script line {i}")
if errors:
for e in errors:
print(f"FAIL: {e}", file=sys.stderr)
print("Rendered with: global.seaweedfs.enableSecurity=true, filer.s3.enabled=true, s3.enabled=true, allInOne.enabled=true", file=sys.stderr)
sys.exit(1)
print("✓ No blank lines in security+S3 command blocks")
PYEOF
echo ""
echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ==="
# Regression test: when enableSecurity=true but *.httpsPort is 0 (the default),
# the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing
# them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which
# makes the HTTP readinessProbe spam "TLS handshake error ... client sent an
# HTTP request to an HTTPS server" into the pod log.
#
# When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted
# together so the opt-in HTTPS listener actually has credentials.
python3 - "$CHART_DIR" <<'PYEOF'
import subprocess, sys, yaml
chart = sys.argv[1]
def render(values):
args = ["helm", "template", "test", chart]
for k, v in values.items():
args += ["--set", f"{k}={v}"]
return subprocess.check_output(args, text=True)
def script_of(manifest, kind_name):
for doc in yaml.safe_load_all(manifest):
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
continue
if doc["metadata"]["name"] != kind_name:
continue
for c in doc["spec"]["template"]["spec"]["containers"]:
cmd = c.get("command", [])
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
return cmd[2]
raise AssertionError(f"no container script for {kind_name}")
cases = [
# (values, workload-name, httpsPort-set?, arg-prefix)
({"global.seaweedfs.enableSecurity": "true",
"s3.enabled": "true"},
"test-seaweedfs-s3", False, ""),
({"global.seaweedfs.enableSecurity": "true",
"s3.enabled": "true",
"s3.httpsPort": "8443"},
"test-seaweedfs-s3", True, ""),
({"global.seaweedfs.enableSecurity": "true",
"filer.s3.enabled": "true"},
"test-seaweedfs-filer", False, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"filer.s3.enabled": "true",
"filer.s3.httpsPort": "8444"},
"test-seaweedfs-filer", True, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"allInOne.enabled": "true",
"allInOne.s3.enabled": "true"},
"test-seaweedfs-all-in-one", False, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"allInOne.enabled": "true",
"allInOne.s3.enabled": "true",
"allInOne.s3.httpsPort": "8445"},
"test-seaweedfs-all-in-one", True, "s3."),
]
failed = False
for values, name, https_on, prefix in cases:
script = script_of(render(values), name)
cert_flag = f"-{prefix}cert.file="
key_flag = f"-{prefix}key.file="
https_flag = f"-{prefix}port.https="
has_cert = cert_flag in script
has_key = key_flag in script
has_https = https_flag in script
label = f"{name} (httpsPort {'set' if https_on else 'unset'})"
if https_on:
if not (has_cert and has_key and has_https):
print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present "
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
failed = True
else:
print(f"✓ {label}: cert/key/https args emitted together")
else:
if has_cert or has_key or has_https:
print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
f"main S3 -port would silently become HTTPS and break HTTP probes "
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
failed = True
else:
print(f"✓ {label}: no TLS args emitted, main -port stays HTTP")
# bash -n: pin down that the rendered script parses. Guards against
# a future helper change that leaves a dangling `\` with nothing
# after it (every current caller already exits cleanly because
# bash treats trailing `\<newline><EOF>` as line-continuation to
# an empty line — but keep the contract explicit).
parse = subprocess.run(["bash", "-n"], input=script, text=True,
capture_output=True)
if parse.returncode != 0:
print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}",
file=sys.stderr)
failed = True
sys.exit(1 if failed else 0)
PYEOF
echo "✅ All template rendering tests passed!"
- name: Create kind cluster
uses: helm/kind-action@v1.14.0
- name: Run chart-testing (install)
run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts
-41
View File
@@ -1,41 +0,0 @@
name: "helm: release"
on:
push:
tags:
- '*'
workflow_dispatch:
permissions:
contents: write
pages: write
packages: write
jobs:
helm-release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Publish Helm charts to github pages
uses: stefanprodan/helm-gh-pages@v1.7.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
charts_dir: k8s/charts
target_dir: helm
branch: gh-pages
helm_version: "3.18.4"
- name: Publish Helm charts to github container registry (ghcr.io)
uses: bitdeps/helm-oci-charts-releaser@v0.1.5
with:
charts_dir: k8s/charts
github_token: ${{ secrets.GITHUB_TOKEN }}
oci_registry: ghcr.io/${{ github.repository_owner }}
oci_username: github-actions
oci_password: ${{ secrets.GITHUB_TOKEN }}
skip_dependencies: true
skip_helm_install: true
skip_gh_release: true
@@ -1,193 +0,0 @@
name: Java Client Integration Tests
on:
push:
branches: [ master ]
paths:
- 'other/java/**'
- 'weed/**'
- '.github/workflows/java_integration_tests.yml'
pull_request:
branches: [ master ]
paths:
- 'other/java/**'
- 'weed/**'
- '.github/workflows/java_integration_tests.yml'
jobs:
test:
name: Java Integration Tests
runs-on: ubuntu-latest
strategy:
matrix:
java: ['11', '17']
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Java
uses: actions/setup-java@v5
with:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- name: Build SeaweedFS
run: |
cd weed
go install -buildvcs=false
weed version
- name: Start SeaweedFS Server
run: |
# Create clean data directory
export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
# Start SeaweedFS with optimized settings for CI
# Include S3 API for s3copier integration tests
weed server -dir="$WEED_DATA_DIR" \
-master.raftHashicorp \
-master.electionTimeout=1s \
-master.volumeSizeLimitMB=100 \
-volume.max=100 \
-volume.preStopSeconds=1 \
-master.peers=none \
-filer -filer.maxMB=64 \
-s3 -s3.port=8333 \
-s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
-s3.allowDeleteBucketNotEmpty=true \
-master.port=9333 \
-volume.port=8080 \
-filer.port=8888 \
-metricsPort=9324 > seaweedfs.log 2>&1 &
SERVER_PID=$!
echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV
echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV
echo "SeaweedFS server started with PID: $SERVER_PID"
- name: Wait for SeaweedFS Components
run: |
echo "Waiting for SeaweedFS components to start..."
# Wait for master
for i in {1..30}; do
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
echo "✓ Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
# Wait for volume
for i in {1..30}; do
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
echo "✓ Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
# Wait for filer
for i in {1..30}; do
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
echo "✓ Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
# Wait for S3 API
for i in {1..30}; do
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
echo "✓ S3 API is ready"
break
fi
echo "Waiting for S3 API... ($i/30)"
sleep 2
done
echo "✓ All SeaweedFS components are ready!"
# Display cluster status
echo "Cluster status:"
curl -s http://localhost:9333/cluster/status | head -20
- name: Build and Install SeaweedFS Client
working-directory: other/java/client
run: |
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Run Client Unit Tests
working-directory: other/java/client
run: |
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
- name: Run Client Integration Tests
working-directory: other/java/client
env:
SEAWEEDFS_TEST_ENABLED: true
run: |
mvn test -Dtest=*IntegrationTest
- name: Run HDFS3 Configuration Tests
working-directory: other/java/hdfs3
run: |
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Run S3 ETag Validation Tests (Issue #7768)
working-directory: other/java/s3copier
env:
S3_ENDPOINT: http://127.0.0.1:8333
S3_ACCESS_KEY: some_access_key1
S3_SECRET_KEY: some_secret_key1
run: |
echo "Running S3 ETag validation tests against $S3_ENDPOINT"
mvn test -Dtest=ETagValidationTest \
-DS3_ENDPOINT=$S3_ENDPOINT \
-DS3_ACCESS_KEY=$S3_ACCESS_KEY \
-DS3_SECRET_KEY=$S3_SECRET_KEY \
-Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Display logs on failure
if: failure()
run: |
echo "=== SeaweedFS Server Log ==="
tail -100 seaweedfs.log || echo "No server log"
echo ""
echo "=== Cluster Status ==="
curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster"
echo ""
echo "=== Process Status ==="
ps aux | grep weed || echo "No weed processes"
- name: Cleanup
if: always()
run: |
# Stop server using stored PID
if [ -n "$SERVER_PID" ]; then
echo "Stopping SeaweedFS server (PID: $SERVER_PID)"
kill -9 $SERVER_PID 2>/dev/null || true
fi
# Fallback: kill any remaining weed processes
pkill -f "weed server" || true
# Clean up data directory
if [ -n "$WEED_DATA_DIR" ]; then
echo "Cleaning up data directory: $WEED_DATA_DIR"
rm -rf "$WEED_DATA_DIR" || true
fi
-58
View File
@@ -1,58 +0,0 @@
name: Java Client Unit Tests
on:
push:
branches: [ master ]
paths:
- 'other/java/**'
- '.github/workflows/java_unit_tests.yml'
pull_request:
branches: [ master ]
paths:
- 'other/java/**'
- '.github/workflows/java_unit_tests.yml'
jobs:
test:
name: Java Unit Tests
runs-on: ubuntu-latest
strategy:
matrix:
java: ['8', '11', '17', '21']
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Java
uses: actions/setup-java@v5
with:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- name: Build and Install SeaweedFS Client
working-directory: other/java/client
run: |
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Run Client Unit Tests
working-directory: other/java/client
run: |
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
- name: Run HDFS3 Configuration Tests
working-directory: other/java/hdfs3
run: |
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Upload Test Reports
if: always()
uses: actions/upload-artifact@v7
with:
name: test-reports-java-${{ matrix.java }}
path: |
other/java/client/target/surefire-reports/
other/java/hdfs3/target/surefire-reports/
-124
View File
@@ -1,124 +0,0 @@
name: "Kafka Quick Test (Load Test with Schema Registry)"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
workflow_dispatch: # Allow manual trigger
concurrency:
group: ${{ github.head_ref }}/kafka-quicktest
cancel-in-progress: true
permissions:
contents: read
jobs:
kafka-client-quicktest:
name: Kafka Client Load Test (Quick)
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
cache-dependency-path: |
**/go.sum
id: go
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Install dependencies
run: |
# Ensure make is available
sudo apt-get update -qq
sudo apt-get install -y make
- name: Validate test setup
working-directory: test/kafka/kafka-client-loadtest
run: |
make validate-setup
- name: Run quick-test
working-directory: test/kafka/kafka-client-loadtest
run: |
# Run the quick-test target which includes:
# 1. Building the gateway
# 2. Starting all services (SeaweedFS, MQ broker, Schema Registry)
# 3. Registering Avro schemas
# 4. Running a 1-minute load test with Avro messages
# Override GOARCH to build for AMD64 (GitHub Actions runners are x86_64)
GOARCH=amd64 make quick-test
env:
# Docker Compose settings
COMPOSE_HTTP_TIMEOUT: 300
DOCKER_CLIENT_TIMEOUT: 300
# Test parameters (set by quick-test, but can override)
TEST_DURATION: 60s
PRODUCER_COUNT: 1
CONSUMER_COUNT: 1
MESSAGE_RATE: 10
VALUE_TYPE: avro
- name: Show test results
if: always()
working-directory: test/kafka/kafka-client-loadtest
run: |
echo "========================================="
echo "Test Results"
echo "========================================="
make show-results || echo "Could not retrieve results"
- name: Show service logs on failure
if: failure()
working-directory: test/kafka/kafka-client-loadtest
run: |
echo "========================================="
echo "Service Logs"
echo "========================================="
echo "Checking running containers..."
docker compose ps || true
echo "========================================="
echo "Master Logs"
echo "========================================="
docker compose logs --tail=100 seaweedfs-master 2>&1 || echo "No master logs available"
echo "========================================="
echo "MQ Broker Logs (Last 100 lines)"
echo "========================================="
docker compose logs --tail=100 seaweedfs-mq-broker 2>&1 || echo "No broker logs available"
echo "========================================="
echo "Kafka Gateway Logs (FULL - Critical for debugging)"
echo "========================================="
docker compose logs kafka-gateway 2>&1 || echo "ERROR: Could not retrieve kafka-gateway logs"
echo "========================================="
echo "Schema Registry Logs (FULL)"
echo "========================================="
docker compose logs schema-registry 2>&1 || echo "ERROR: Could not retrieve schema-registry logs"
echo "========================================="
echo "Load Test Logs"
echo "========================================="
docker compose logs --tail=100 kafka-client-loadtest 2>&1 || echo "No loadtest logs available"
- name: Cleanup
if: always()
working-directory: test/kafka/kafka-client-loadtest
run: |
# Stop containers first
docker compose --profile loadtest --profile monitoring down -v --remove-orphans || true
# Clean up data with sudo to handle Docker root-owned files
sudo rm -rf data/* || true
# Clean up binary
rm -f weed-linux-* || true
-825
View File
@@ -1,825 +0,0 @@
name: "Kafka Gateway Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
concurrency:
group: ${{ github.head_ref }}/kafka-tests
cancel-in-progress: true
# Force different runners for better isolation
env:
FORCE_RUNNER_SEPARATION: true
permissions:
contents: read
jobs:
kafka-unit-tests:
name: Kafka Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
fail-fast: false
matrix:
container-id: [unit-tests-1]
container:
image: golang:1.24-alpine
options: --cpus 1.0 --memory 1g --hostname kafka-unit-${{ matrix.container-id }}
env:
GOMAXPROCS: 1
CGO_ENABLED: 0
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code
uses: actions/checkout@v6
- name: Setup Container Environment
run: |
apk add --no-cache git
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
- name: Get dependencies
run: |
cd test/kafka
go mod download
go mod tidy
- name: Run Kafka Gateway Unit Tests
run: |
cd test/kafka
# Set process limits for container isolation
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
ulimit -u 100 || echo "Warning: Could not set process limit"
go test -v -timeout 10s ./unit/...
kafka-integration-tests:
name: Kafka Integration Tests (Critical)
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
fail-fast: false
matrix:
container-id: [integration-1]
container:
image: golang:1.24-alpine
options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }}
env:
GOMAXPROCS: 2
CGO_ENABLED: 0
KAFKA_TEST_ISOLATION: "true"
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code
uses: actions/checkout@v6
- name: Setup Integration Container Environment
run: |
apk add --no-cache git procps
ulimit -n 2048 || echo "Warning: Could not set file descriptor limit"
- name: Get dependencies
run: |
cd test/kafka
go mod download
go mod tidy
- name: Run Integration Tests
run: |
cd test/kafka
# Higher limits for integration tests
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
ulimit -u 200 || echo "Warning: Could not set process limit"
go test -v -timeout 90s ./integration/...
env:
GOMAXPROCS: 2
kafka-e2e-tests:
name: Kafka End-to-End Tests (with SMQ)
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
container-id: [e2e-1]
container:
image: golang:1.24-alpine
options: --cpus 2.0 --memory 2g --hostname kafka-e2e-${{ matrix.container-id }}
env:
GOMAXPROCS: 2
CGO_ENABLED: 0
KAFKA_E2E_ISOLATION: "true"
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
cache-dependency-path: |
**/go.sum
id: go
- name: Setup E2E Container Environment
run: |
apk add --no-cache git procps curl netcat-openbsd
ulimit -n 2048 || echo "Warning: Could not set file descriptor limit"
- name: Warm Go module cache
run: |
# Warm cache for root module
go mod download || true
# Warm cache for kafka test module
cd test/kafka
go mod download || true
- name: Get dependencies
run: |
cd test/kafka
# Use go mod download with timeout to prevent hanging
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
go mod tidy
- name: Build and start SeaweedFS MQ
run: |
set -e
cd $GITHUB_WORKSPACE
# Build weed binary
go build -o /usr/local/bin/weed ./weed
# Start SeaweedFS components with MQ brokers
export WEED_DATA_DIR=/tmp/seaweedfs-e2e-$RANDOM
mkdir -p "$WEED_DATA_DIR"
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
nohup weed -v 1 server \
-ip="127.0.0.1" \
-ip.bind="0.0.0.0" \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp \
-master.port=9333 \
-volume.port=8081 \
-filer.port=8888 \
-filer=true \
-metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 &
# Wait for master to be ready
for i in $(seq 1 30); do
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
echo "SeaweedFS master HTTP is up"; break
fi
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
done
# Wait for master gRPC to be ready (this is what broker discovery uses)
echo "Waiting for master gRPC port..."
for i in $(seq 1 30); do
if nc -z 127.0.0.1 19333; then
echo "✓ SeaweedFS master gRPC is up (port 19333)"
break
fi
echo " Waiting for master gRPC... ($i/30)"; sleep 1
done
# Give server time to initialize all components including gRPC services
echo "Waiting for SeaweedFS components to initialize..."
sleep 15
# Additional wait specifically for gRPC services to be ready for streaming
echo "Allowing extra time for master gRPC streaming services to initialize..."
sleep 10
# Start MQ broker with maximum verbosity for debugging
echo "Starting MQ broker..."
nohup weed -v 3 mq.broker \
-master="127.0.0.1:9333" \
-ip="127.0.0.1" \
-port=17777 \
-logFlushInterval=0 \
> /tmp/weed-mq-broker.log 2>&1 &
# Wait for broker to be ready with better error reporting
sleep 15
broker_ready=false
for i in $(seq 1 20); do
if nc -z 127.0.0.1 17777; then
echo "SeaweedFS MQ broker is up"
broker_ready=true
break
fi
echo "Waiting for MQ broker... ($i/20)"; sleep 1
done
# Give broker additional time to register with master
if [ "$broker_ready" = true ]; then
echo "Allowing broker to register with master..."
sleep 30
# Check if broker is properly registered by querying cluster nodes
echo "Cluster status after broker registration:"
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
echo "Checking cluster topology (includes registered components):"
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
echo "Verifying broker discovery via master client debug:"
echo "If broker registration is successful, it should appear in dir status"
echo "Testing gRPC connectivity with weed binary:"
echo "This simulates what the gateway does during broker discovery..."
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
echo "Shell test results:"
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
fi
# Check if broker failed to start and show logs
if [ "$broker_ready" = false ]; then
echo "ERROR: MQ broker failed to start. Broker logs:"
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
echo "Server logs:"
tail -20 /tmp/weed-server.log || echo "No server logs found"
exit 1
fi
- name: Run End-to-End Tests
run: |
cd test/kafka
# Higher limits for E2E tests
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
ulimit -u 200 || echo "Warning: Could not set process limit"
# Allow additional time for all background processes to settle
echo "Allowing additional settlement time for SeaweedFS ecosystem..."
sleep 15
# Run tests and capture result
if ! go test -v -timeout 180s ./e2e/...; then
echo "========================================="
echo "Tests failed! Showing debug information:"
echo "========================================="
echo "Server logs (last 50 lines):"
tail -50 /tmp/weed-server.log || echo "No server logs"
echo "========================================="
echo "Broker logs (last 50 lines):"
tail -50 /tmp/weed-mq-broker.log || echo "No broker logs"
echo "========================================="
exit 1
fi
env:
GOMAXPROCS: 2
SEAWEEDFS_MASTERS: 127.0.0.1:9333
kafka-consumer-group-tests:
name: Kafka Consumer Group Tests (Highly Isolated)
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
container-id: [consumer-group-1]
container:
image: golang:1.24-alpine
options: --cpus 1.0 --memory 2g --ulimit nofile=512:512 --hostname kafka-consumer-${{ matrix.container-id }}
env:
GOMAXPROCS: 1
CGO_ENABLED: 0
KAFKA_CONSUMER_ISOLATION: "true"
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
cache-dependency-path: |
**/go.sum
id: go
- name: Setup Consumer Group Container Environment
run: |
apk add --no-cache git procps curl netcat-openbsd
ulimit -n 256 || echo "Warning: Could not set file descriptor limit"
- name: Warm Go module cache
run: |
# Warm cache for root module
go mod download || true
# Warm cache for kafka test module
cd test/kafka
go mod download || true
- name: Get dependencies
run: |
cd test/kafka
# Use go mod download with timeout to prevent hanging
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
go mod tidy
- name: Build and start SeaweedFS MQ
run: |
set -e
cd $GITHUB_WORKSPACE
# Build weed binary
go build -o /usr/local/bin/weed ./weed
# Start SeaweedFS components with MQ brokers
export WEED_DATA_DIR=/tmp/seaweedfs-mq-$RANDOM
mkdir -p "$WEED_DATA_DIR"
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
nohup weed -v 1 server \
-ip="127.0.0.1" \
-ip.bind="0.0.0.0" \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp \
-master.port=9333 \
-volume.port=8081 \
-filer.port=8888 \
-filer=true \
-metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 &
# Wait for master to be ready
for i in $(seq 1 30); do
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
echo "SeaweedFS master HTTP is up"; break
fi
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
done
# Wait for master gRPC to be ready (this is what broker discovery uses)
echo "Waiting for master gRPC port..."
for i in $(seq 1 30); do
if nc -z 127.0.0.1 19333; then
echo "✓ SeaweedFS master gRPC is up (port 19333)"
break
fi
echo " Waiting for master gRPC... ($i/30)"; sleep 1
done
# Give server time to initialize all components including gRPC services
echo "Waiting for SeaweedFS components to initialize..."
sleep 15
# Additional wait specifically for gRPC services to be ready for streaming
echo "Allowing extra time for master gRPC streaming services to initialize..."
sleep 10
# Start MQ broker with maximum verbosity for debugging
echo "Starting MQ broker..."
nohup weed -v 3 mq.broker \
-master="127.0.0.1:9333" \
-ip="127.0.0.1" \
-port=17777 \
-logFlushInterval=0 \
> /tmp/weed-mq-broker.log 2>&1 &
# Wait for broker to be ready with better error reporting
sleep 15
broker_ready=false
for i in $(seq 1 20); do
if nc -z 127.0.0.1 17777; then
echo "SeaweedFS MQ broker is up"
broker_ready=true
break
fi
echo "Waiting for MQ broker... ($i/20)"; sleep 1
done
# Give broker additional time to register with master
if [ "$broker_ready" = true ]; then
echo "Allowing broker to register with master..."
sleep 30
# Check if broker is properly registered by querying cluster nodes
echo "Cluster status after broker registration:"
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
echo "Checking cluster topology (includes registered components):"
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
echo "Verifying broker discovery via master client debug:"
echo "If broker registration is successful, it should appear in dir status"
echo "Testing gRPC connectivity with weed binary:"
echo "This simulates what the gateway does during broker discovery..."
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
echo "Shell test results:"
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
fi
# Check if broker failed to start and show logs
if [ "$broker_ready" = false ]; then
echo "ERROR: MQ broker failed to start. Broker logs:"
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
echo "Server logs:"
tail -20 /tmp/weed-server.log || echo "No server logs found"
exit 1
fi
- name: Run Consumer Group Tests
run: |
cd test/kafka
# Test consumer group functionality with explicit timeout
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
ulimit -u 100 || echo "Warning: Could not set process limit"
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
env:
GOMAXPROCS: 1
SEAWEEDFS_MASTERS: 127.0.0.1:9333
kafka-client-compatibility:
name: Kafka Client Compatibility (with SMQ)
runs-on: ubuntu-latest
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
container-id: [client-compat-1]
container:
image: golang:1.24-alpine
options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }}
env:
GOMAXPROCS: 1
CGO_ENABLED: 0
KAFKA_CLIENT_ISOLATION: "true"
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
cache-dependency-path: |
**/go.sum
id: go
- name: Setup Client Container Environment
run: |
apk add --no-cache git procps curl netcat-openbsd
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
- name: Warm Go module cache
run: |
# Warm cache for root module
go mod download || true
# Warm cache for kafka test module
cd test/kafka
go mod download || true
- name: Get dependencies
run: |
cd test/kafka
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
go mod tidy
- name: Build and start SeaweedFS MQ
run: |
set -e
cd $GITHUB_WORKSPACE
# Build weed binary
go build -o /usr/local/bin/weed ./weed
# Start SeaweedFS components with MQ brokers
export WEED_DATA_DIR=/tmp/seaweedfs-client-$RANDOM
mkdir -p "$WEED_DATA_DIR"
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
nohup weed -v 1 server \
-ip="127.0.0.1" \
-ip.bind="0.0.0.0" \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp \
-master.port=9333 \
-volume.port=8081 \
-filer.port=8888 \
-filer=true \
-metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 &
# Wait for master to be ready
for i in $(seq 1 30); do
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
echo "SeaweedFS master HTTP is up"; break
fi
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
done
# Wait for master gRPC to be ready (this is what broker discovery uses)
echo "Waiting for master gRPC port..."
for i in $(seq 1 30); do
if nc -z 127.0.0.1 19333; then
echo "✓ SeaweedFS master gRPC is up (port 19333)"
break
fi
echo " Waiting for master gRPC... ($i/30)"; sleep 1
done
# Give server time to initialize all components including gRPC services
echo "Waiting for SeaweedFS components to initialize..."
sleep 15
# Additional wait specifically for gRPC services to be ready for streaming
echo "Allowing extra time for master gRPC streaming services to initialize..."
sleep 10
# Start MQ broker with maximum verbosity for debugging
echo "Starting MQ broker..."
nohup weed -v 3 mq.broker \
-master="127.0.0.1:9333" \
-ip="127.0.0.1" \
-port=17777 \
-logFlushInterval=0 \
> /tmp/weed-mq-broker.log 2>&1 &
# Wait for broker to be ready with better error reporting
sleep 15
broker_ready=false
for i in $(seq 1 20); do
if nc -z 127.0.0.1 17777; then
echo "SeaweedFS MQ broker is up"
broker_ready=true
break
fi
echo "Waiting for MQ broker... ($i/20)"; sleep 1
done
# Give broker additional time to register with master
if [ "$broker_ready" = true ]; then
echo "Allowing broker to register with master..."
sleep 30
# Check if broker is properly registered by querying cluster nodes
echo "Cluster status after broker registration:"
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
echo "Checking cluster topology (includes registered components):"
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
echo "Verifying broker discovery via master client debug:"
echo "If broker registration is successful, it should appear in dir status"
echo "Testing gRPC connectivity with weed binary:"
echo "This simulates what the gateway does during broker discovery..."
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
echo "Shell test results:"
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
fi
# Check if broker failed to start and show logs
if [ "$broker_ready" = false ]; then
echo "ERROR: MQ broker failed to start. Broker logs:"
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
echo "Server logs:"
tail -20 /tmp/weed-server.log || echo "No server logs found"
exit 1
fi
- name: Run Client Compatibility Tests
run: |
cd test/kafka
go test -v -run "^TestClientCompatibility" -timeout 180s ./integration/...
env:
GOMAXPROCS: 1
SEAWEEDFS_MASTERS: 127.0.0.1:9333
kafka-smq-integration-tests:
name: Kafka SMQ Integration Tests (Full Stack)
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
container-id: [smq-integration-1]
container:
image: golang:1.24-alpine
options: --cpus 1.0 --memory 2g --hostname kafka-smq-${{ matrix.container-id }}
env:
GOMAXPROCS: 1
CGO_ENABLED: 0
KAFKA_SMQ_INTEGRATION: "true"
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
cache-dependency-path: |
**/go.sum
id: go
- name: Setup SMQ Integration Container Environment
run: |
apk add --no-cache git procps curl netcat-openbsd
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
- name: Warm Go module cache
run: |
# Warm cache for root module
go mod download || true
# Warm cache for kafka test module
cd test/kafka
go mod download || true
- name: Get dependencies
run: |
cd test/kafka
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
go mod tidy
- name: Build and start SeaweedFS MQ
run: |
set -e
cd $GITHUB_WORKSPACE
# Build weed binary
go build -o /usr/local/bin/weed ./weed
# Start SeaweedFS components with MQ brokers
export WEED_DATA_DIR=/tmp/seaweedfs-smq-$RANDOM
mkdir -p "$WEED_DATA_DIR"
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
nohup weed -v 1 server \
-ip="127.0.0.1" \
-ip.bind="0.0.0.0" \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp \
-master.port=9333 \
-volume.port=8081 \
-filer.port=8888 \
-filer=true \
-metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 &
# Wait for master to be ready
for i in $(seq 1 30); do
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
echo "SeaweedFS master HTTP is up"; break
fi
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
done
# Wait for master gRPC to be ready (this is what broker discovery uses)
echo "Waiting for master gRPC port..."
for i in $(seq 1 30); do
if nc -z 127.0.0.1 19333; then
echo "✓ SeaweedFS master gRPC is up (port 19333)"
break
fi
echo " Waiting for master gRPC... ($i/30)"; sleep 1
done
# Give server time to initialize all components including gRPC services
echo "Waiting for SeaweedFS components to initialize..."
sleep 15
# Additional wait specifically for gRPC services to be ready for streaming
echo "Allowing extra time for master gRPC streaming services to initialize..."
sleep 10
# Start MQ broker with maximum verbosity for debugging
echo "Starting MQ broker..."
nohup weed -v 3 mq.broker \
-master="127.0.0.1:9333" \
-ip="127.0.0.1" \
-port=17777 \
-logFlushInterval=0 \
> /tmp/weed-mq-broker.log 2>&1 &
# Wait for broker to be ready with better error reporting
sleep 15
broker_ready=false
for i in $(seq 1 20); do
if nc -z 127.0.0.1 17777; then
echo "SeaweedFS MQ broker is up"
broker_ready=true
break
fi
echo "Waiting for MQ broker... ($i/20)"; sleep 1
done
# Give broker additional time to register with master
if [ "$broker_ready" = true ]; then
echo "Allowing broker to register with master..."
sleep 30
# Check if broker is properly registered by querying cluster nodes
echo "Cluster status after broker registration:"
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
echo "Checking cluster topology (includes registered components):"
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
echo "Verifying broker discovery via master client debug:"
echo "If broker registration is successful, it should appear in dir status"
echo "Testing gRPC connectivity with weed binary:"
echo "This simulates what the gateway does during broker discovery..."
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
echo "Shell test results:"
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
fi
# Check if broker failed to start and show logs
if [ "$broker_ready" = false ]; then
echo "ERROR: MQ broker failed to start. Broker logs:"
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
echo "Server logs:"
tail -20 /tmp/weed-server.log || echo "No server logs found"
exit 1
fi
- name: Run SMQ Integration Tests
run: |
cd test/kafka
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
ulimit -u 100 || echo "Warning: Could not set process limit"
# Run the dedicated SMQ integration tests
go test -v -run "^TestSMQIntegration" -timeout 180s ./integration/...
env:
GOMAXPROCS: 1
SEAWEEDFS_MASTERS: 127.0.0.1:9333
kafka-protocol-tests:
name: Kafka Protocol Tests (Isolated)
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
fail-fast: false
matrix:
container-id: [protocol-1]
container:
image: golang:1.24-alpine
options: --cpus 1.0 --memory 1g --tmpfs /tmp:exec --hostname kafka-protocol-${{ matrix.container-id }}
env:
GOMAXPROCS: 1
CGO_ENABLED: 0
KAFKA_PROTOCOL_ISOLATION: "true"
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code
uses: actions/checkout@v6
- name: Setup Protocol Container Environment
run: |
apk add --no-cache git procps
# Ensure proper permissions for test execution
chmod -R 755 /tmp || true
export TMPDIR=/tmp
export GOCACHE=/tmp/go-cache
mkdir -p $GOCACHE
chmod 755 $GOCACHE
- name: Get dependencies
run: |
cd test/kafka
go mod download
go mod tidy
- name: Run Protocol Tests
run: |
cd test/kafka
export TMPDIR=/tmp
export GOCACHE=/tmp/go-cache
# Run protocol tests from the weed/mq/kafka directory since they test the protocol implementation
cd ../../weed/mq/kafka
go test -v -run "^Test.*" -timeout 10s ./...
env:
GOMAXPROCS: 1
TMPDIR: /tmp
GOCACHE: /tmp/go-cache
-140
View File
@@ -1,140 +0,0 @@
name: "KMS Tests"
on:
pull_request:
paths:
- 'weed/kms/**'
- 'weed/s3api/s3_sse_*.go'
- 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'test/kms/**'
- '.github/workflows/kms-tests.yml'
push:
branches: [ master, main ]
paths:
- 'weed/kms/**'
- 'weed/s3api/s3_sse_*.go'
- 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'test/kms/**'
concurrency:
group: ${{ github.head_ref || github.ref }}-kms-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
kms-provider-tests:
name: KMS Provider Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run KMS provider integration tests
timeout-minutes: 15
working-directory: test/kms
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
docker --version
make test-provider-ci
- name: Show OpenBao logs on failure
if: failure()
run: |
echo "=== OpenBao Container Logs ==="
docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
echo "=== Setup Logs ==="
cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found"
- name: Cleanup
if: always()
working-directory: test/kms
run: |
make stop-openbao-ci || true
s3-kms-e2e-tests:
name: S3 KMS End-to-End Tests
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 KMS end-to-end tests
timeout-minutes: 20
working-directory: test/kms
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
docker --version
aws --version
make test-s3-kms-ci
- name: Show logs on failure
if: failure()
working-directory: test/kms
run: |
echo "=== OpenBao Container Logs ==="
cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
echo "=== SeaweedFS Server Logs ==="
tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
echo "=== Setup Logs ==="
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
echo "=== Process Information ==="
ps aux | grep -E "(weed|test)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-kms-e2e-logs
path: |
/tmp/seaweedfs-kms-mini.log
/tmp/openbao-ci-container.log
/tmp/weed-kms-ci-setup.log
retention-days: 3
- name: Cleanup
if: always()
working-directory: test/kms
run: |
make stop-seaweedfs-ci || true
make stop-openbao-ci || true
make clean-ci || true
@@ -1,91 +0,0 @@
name: "Metadata Subscribe Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/filer/**'
- 'weed/pb/filer_pb/**'
- 'weed/util/log_buffer/**'
- 'weed/server/filer_grpc_server_sub_meta.go'
- 'weed/command/filer_backup.go'
- 'test/metadata_subscribe/**'
- '.github/workflows/metadata-subscribe-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/filer/**'
- 'weed/pb/filer_pb/**'
- 'weed/util/log_buffer/**'
- 'weed/server/filer_grpc_server_sub_meta.go'
- 'weed/command/filer_backup.go'
- 'test/metadata_subscribe/**'
- '.github/workflows/metadata-subscribe-tests.yml'
concurrency:
group: ${{ github.head_ref }}/metadata-subscribe-tests
cancel-in-progress: true
permissions:
contents: read
env:
TEST_TIMEOUT: '10m'
jobs:
metadata-subscribe-integration:
name: Metadata Subscribe Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Run Metadata Subscribe Integration Tests
run: |
cd test/metadata_subscribe
echo "Running Metadata Subscribe integration tests..."
echo "============================================"
# Run tests with verbose output
go test -v -timeout=${{ env.TEST_TIMEOUT }} ./...
echo "============================================"
echo "Metadata Subscribe integration tests completed"
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: metadata-subscribe-test-logs
path: |
/tmp/seaweedfs_*
retention-days: 7
- name: Test Summary
if: always()
run: |
echo "## Metadata Subscribe Integration Test Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **Basic Subscription**: Subscribe to metadata changes and receive events" >> $GITHUB_STEP_SUMMARY
echo "- **Single-Filer No Stall**: Regression test for issue #4977" >> $GITHUB_STEP_SUMMARY
echo "- **Resume from Disk**: Verify subscription can resume from persisted logs" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Related Issues" >> $GITHUB_STEP_SUMMARY
echo "- [#4977](https://github.com/seaweedfs/seaweedfs/issues/4977): filer.backup synchronisation stall" >> $GITHUB_STEP_SUMMARY
-77
View File
@@ -1,77 +0,0 @@
name: "Multi-Master Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/server/raft_*.go'
- 'weed/topology/**'
- 'test/multi_master/**'
- 'test/testutil/**'
- '.github/workflows/multi-master-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/server/raft_*.go'
- 'weed/topology/**'
- 'test/multi_master/**'
- 'test/testutil/**'
- '.github/workflows/multi-master-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/multi-master-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
multi-master-failover-tests:
name: Multi-Master Failover Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Run multi-master failover tests
# The tests in test/multi_master spin up their own 3-node master raft
# cluster (using the freshly-installed `weed` binary) and exercise
# leader-election, failover and recovery scenarios. The shared
# test/testutil port-allocator regression test runs alongside since it
# is a prerequisite for the cluster fixtures.
run: |
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
- name: Collect server logs on failure
if: failure()
run: |
# test/multi_master/cluster.go creates per-test dirs via
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
# node's log into <baseDir>/logs/master*.log.
echo "Collecting per-node master logs from temp directories..."
mkdir -p /tmp/multi-master-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
echo "Found test directory: $dir"
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
done
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: multi-master-test-logs
path: /tmp/multi-master-logs/
retention-days: 7
-137
View File
@@ -1,137 +0,0 @@
name: "NFS Integration Tests"
on:
push:
branches: [ master, main ]
paths:
- 'weed/server/nfs/**'
- 'weed/command/nfs.go'
- 'weed/filer/filer_inode.go'
- 'weed/filer/filer_inode_index.go'
- 'weed/filer/filerstore_wrapper.go'
- 'weed/server/filer_grpc_server_rename.go'
- 'test/nfs/**'
- '.github/workflows/nfs-tests.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/server/nfs/**'
- 'weed/command/nfs.go'
- 'weed/filer/filer_inode.go'
- 'weed/filer/filer_inode_index.go'
- 'weed/filer/filerstore_wrapper.go'
- 'weed/server/filer_grpc_server_rename.go'
- 'test/nfs/**'
- '.github/workflows/nfs-tests.yml'
concurrency:
group: ${{ github.head_ref }}/nfs-tests
cancel-in-progress: true
permissions:
contents: read
env:
TEST_TIMEOUT: '15m'
jobs:
nfs-integration:
name: NFS Integration Testing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Run NFS Integration Tests
run: |
cd test/nfs
echo "Running NFS integration tests..."
echo "============================================"
# Install test dependencies
go mod download
# Run the protocol-layer tests. The kernel-mount tests require root
# for mount(2) and are exercised in their own privileged step below;
# skip them here so a "skipped because not root" line doesn't show
# up as noise on every CI run.
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
echo "============================================"
echo "NFS integration tests completed"
- name: Install kernel NFS client
run: |
# nfs-common provides mount.nfs; netbase provides /etc/protocols
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
sudo apt-get update
sudo apt-get install -y nfs-common netbase
- name: Run kernel-mount E2E tests
run: |
cd test/nfs
echo "Running kernel-mount end-to-end tests..."
echo "These mount the running 'weed nfs' subprocess via the actual"
echo "Linux NFS client to catch protocol regressions invisible to"
echo "the go-nfs-client-based tests above."
echo "============================================"
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
# binary that test/nfs/framework.go locates via $PATH) resolve
# correctly under sudo, and pass through the Go module/cache dirs
# so we don't redownload modules under root.
sudo env "PATH=$PATH" \
GOMODCACHE="$(go env GOMODCACHE)" \
GOCACHE="$(go env GOCACHE)" \
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
echo "============================================"
echo "Kernel-mount E2E tests completed"
- name: Test Summary
if: always()
run: |
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Harness" >> $GITHUB_STEP_SUMMARY
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
-118
View File
@@ -1,118 +0,0 @@
name: "pjdfstest POSIX Compliance"
on:
push:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'test/pjdfstest/**'
- '.github/workflows/pjdfstest.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'test/pjdfstest/**'
- '.github/workflows/pjdfstest.yml'
workflow_dispatch:
concurrency:
group: pjdfstest/${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
pjdfstest:
name: pjdfstest
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Start local Docker registry
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Build weed race binary
run: |
cd docker
make binary_race
- name: Build SeaweedFS e2e image
uses: docker/build-push-action@v7
with:
context: docker
file: docker/Dockerfile.e2e
tags: localhost:5000/chrislusf/seaweedfs:e2e
push: true
cache-from: type=gha,scope=pjdfstest-e2e
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
- name: Tag e2e image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:e2e
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
- name: Build pjdfstest image
uses: docker/build-push-action@v7
with:
context: test/pjdfstest
build-contexts: |
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
push: true
cache-from: type=gha,scope=pjdfstest-harness
cache-to: type=gha,mode=max,scope=pjdfstest-harness
- name: Tag pjdfstest image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
- name: Start SeaweedFS cluster
run: |
docker compose -f test/pjdfstest/docker-compose.yml up --wait
- name: Run pjdfstest
run: |
set -o pipefail
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
- name: Collect logs
if: always()
run: |
mkdir -p /tmp/pjdfstest-docker-logs
for svc in master volume filer mount; do
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
done
- name: Tear down
if: always()
run: |
docker compose -f test/pjdfstest/docker-compose.yml down -v
- name: Upload logs
if: always()
uses: actions/upload-artifact@v7
with:
name: pjdfstest-results
path: |
/tmp/pjdfstest-output.log
/tmp/pjdfstest-docker-logs/
retention-days: 7
-39
View File
@@ -1,39 +0,0 @@
name: "Plugin Worker Integration Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
permissions:
contents: read
jobs:
plugin-worker:
name: "Plugin Worker: ${{ matrix.worker }}"
runs-on: ubuntu-22.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
include:
- worker: erasure_coding
path: test/plugin_workers/erasure_coding
- worker: vacuum
path: test/plugin_workers/vacuum
- worker: volume_balance
path: test/plugin_workers/volume_balance
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.26
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Run plugin worker tests
run: go test -v ./${{ matrix.path }}
-73
View File
@@ -1,73 +0,0 @@
name: "PostgreSQL Gateway Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
concurrency:
group: ${{ github.head_ref }}/postgres-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
postgres-basic-tests:
name: PostgreSQL Basic Tests
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
run:
working-directory: test/postgres
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code
uses: actions/checkout@v6
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Cache Docker layers
uses: actions/cache@v5
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-postgres-
- name: Start PostgreSQL Gateway Services
run: |
make dev-start
sleep 10
- name: Run Basic Connectivity Test
run: |
make test-basic
- name: Run PostgreSQL Client Tests
run: |
make test-client
- name: Save logs
if: always()
run: |
docker compose logs > postgres-output.log || true
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
with:
name: postgres-logs
path: test/postgres/postgres-output.log
- name: Cleanup
if: always()
run: |
make clean || true
@@ -1,242 +0,0 @@
name: "Rust Volume Server Tests"
on:
pull_request:
branches: [ master ]
paths:
- 'seaweed-volume/**'
- 'test/volume_server/**'
- 'weed/pb/volume_server.proto'
- 'weed/pb/volume_server_pb/**'
- '.github/workflows/rust-volume-server-tests.yml'
push:
branches: [ master, main ]
paths:
- 'seaweed-volume/**'
- 'test/volume_server/**'
- 'weed/pb/volume_server.proto'
- 'weed/pb/volume_server_pb/**'
- '.github/workflows/rust-volume-server-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
rust-unit-tests:
name: Rust Unit Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build Rust volume server
run: cd seaweed-volume && cargo build --release
- name: Run Rust unit tests
run: cd seaweed-volume && cargo test
rust-integration-tests:
name: Rust Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build Go weed binary
run: |
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
- name: Run integration tests
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
run: |
echo "Running Rust volume server integration tests..."
go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
- name: Collect logs on failure
if: failure()
run: |
mkdir -p /tmp/rust-volume-server-it-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: rust-volume-server-integration-test-logs
path: /tmp/rust-volume-server-it-logs/
if-no-files-found: warn
retention-days: 7
- name: Test summary
if: always()
run: |
echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
rust-volume-go-tests:
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
runs-on: ubuntu-22.04
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
test-type: [grpc, http]
shard: [1, 2, 3]
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build Go weed binary
run: |
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
- name: Run volume server integration tests with Rust volume
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
VOLUME_SERVER_IMPL: rust
run: |
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
- name: Collect logs on failure
if: failure()
run: |
mkdir -p /tmp/rust-volume-go-test-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
path: /tmp/rust-volume-go-test-logs/
if-no-files-found: warn
retention-days: 7
- name: Test summary
if: always()
run: |
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
-166
View File
@@ -1,166 +0,0 @@
name: "rust: build dev volume server binaries"
on:
push:
branches: [ master ]
paths:
- 'seaweed-volume/**'
- '.github/workflows/rust_binaries_dev.yml'
permissions:
contents: read
jobs:
cleanup:
permissions:
contents: write
runs-on: ubuntu-latest
steps:
- name: Delete old Rust volume dev assets
uses: mknejp/delete-release-assets@v1
continue-on-error: true
with:
token: ${{ github.token }}
tag: dev
fail-if-no-assets: false
assets: |
weed-volume-*
build-rust-volume-dev-linux:
permissions:
contents: write
needs: cleanup
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
asset_suffix: linux-amd64
steps:
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-dev-${{ matrix.target }}-
- name: Set BUILD_TIME
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: cd seaweed-volume && cargo build --release
- name: Package large disk binary
run: |
cp seaweed-volume/target/release/weed-volume weed-volume-large-disk
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
rm weed-volume-large-disk
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: cd seaweed-volume && cargo build --release --no-default-features
- name: Package normal binary
run: |
cp seaweed-volume/target/release/weed-volume weed-volume-normal
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
rm weed-volume-normal
- name: Upload dev release assets
uses: softprops/action-gh-release@v3
with:
tag_name: dev
prerelease: true
files: |
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build-rust-volume-dev-darwin:
permissions:
contents: write
needs: build-rust-volume-dev-linux
runs-on: macos-latest
strategy:
matrix:
include:
- target: aarch64-apple-darwin
asset_suffix: darwin-arm64
- target: x86_64-apple-darwin
asset_suffix: darwin-amd64
steps:
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: brew install protobuf
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-dev-${{ matrix.target }}-
- name: Set BUILD_TIME
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }}
- name: Package large disk binary
run: |
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
rm weed-volume-large-disk
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features
- name: Package normal binary
run: |
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
rm weed-volume-normal
- name: Upload dev release assets
uses: softprops/action-gh-release@v3
with:
tag_name: dev
prerelease: true
files: |
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-253
View File
@@ -1,253 +0,0 @@
name: "rust: build versioned volume server binaries"
on:
push:
tags:
- '*'
workflow_dispatch:
permissions:
contents: read
jobs:
build-rust-volume-linux:
permissions:
contents: write
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
asset_suffix: linux_amd64
- target: aarch64-unknown-linux-gnu
asset_suffix: linux_arm64
cross: true
steps:
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install cross-compilation tools
if: matrix.cross
run: |
sudo dpkg --add-architecture arm64
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-release-${{ matrix.target }}-
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }}
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
- name: Package binaries
run: |
# Large disk (default, 5bytes feature)
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
rm weed-volume-large-disk
# Normal volume size
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
rm weed-volume-normal
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
with:
files: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.asset_suffix }}
path: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz
build-rust-volume-darwin:
permissions:
contents: write
runs-on: macos-latest
strategy:
matrix:
include:
- target: x86_64-apple-darwin
asset_suffix: darwin_amd64
- target: aarch64-apple-darwin
asset_suffix: darwin_arm64
steps:
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: brew install protobuf
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-release-${{ matrix.target }}-
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }}
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
- name: Package binaries
run: |
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
rm weed-volume-large-disk
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
rm weed-volume-normal
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
with:
files: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.asset_suffix }}
path: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz
build-rust-volume-windows:
permissions:
contents: write
runs-on: windows-latest
steps:
- uses: actions/checkout@v6
- name: Install protobuf compiler
run: choco install protoc -y
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-release-windows-
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --no-default-features
- name: Package binaries
shell: bash
run: |
cp seaweed-volume/target/release/weed-volume.exe weed-volume-large-disk.exe
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
rm weed-volume-large-disk.exe
cp seaweed-volume/target/release/weed-volume.exe weed-volume-normal.exe
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
rm weed-volume-normal.exe
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
with:
files: |
weed-volume_large_disk_windows_amd64.zip
weed-volume_windows_amd64.zip
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-windows_amd64
path: |
weed-volume_large_disk_windows_amd64.zip
weed-volume_windows_amd64.zip
-129
View File
@@ -1,129 +0,0 @@
name: "S3 ETag and ACL Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/etag*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/etag/**'
- 'test/s3/acl/**'
- '.github/workflows/s3-etag-acl-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/etag*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/etag/**'
- 'test/s3/acl/**'
- '.github/workflows/s3-etag-acl-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-etag-acl-tests:
name: S3 ETag + ACL Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Start weed mini (S3 on :8333)
run: |
mkdir -p /tmp/seaweedfs-etag-acl
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
{
"identities": [
{
"name": "admin",
"credentials": [
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
],
"actions": ["Admin", "Read", "Write"]
}
]
}
JSON
AWS_ACCESS_KEY_ID=some_access_key1 \
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
weed mini \
-dir=/tmp/seaweedfs-etag-acl \
-s3.port=8333 \
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
-ip=127.0.0.1 \
> /tmp/weed-mini.log 2>&1 &
echo $! > /tmp/weed-mini.pid
# Wait for the S3 endpoint to come up (returns 403 unauth before any
# request is signed; that's fine — it means the server is listening).
for i in $(seq 1 30); do
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
echo "weed mini is ready"
exit 0
fi
sleep 1
done
echo "weed mini failed to start within 30s"
tail -50 /tmp/weed-mini.log
exit 1
- name: Run ETag tests
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
# SDK for Java v2 rejects the latter on the PutObject path.
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/etag/...
- name: Run ACL versioning tests
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
# PutObjectAcl with and without versionId, modifying ACLs on different
# versions independently.
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/acl/...
- name: Stop weed mini
if: always()
run: |
if [ -f /tmp/weed-mini.pid ]; then
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
fi
- name: Show server log on failure
if: failure()
run: |
echo "=== weed mini log (last 200 lines) ==="
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
- name: Archive log
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-etag-acl-server-log
path: /tmp/weed-mini.log
retention-days: 3
@@ -1,72 +0,0 @@
name: "S3 Authenticated Integration Tests"
on:
pull_request:
concurrency:
group: ${{ github.head_ref }}/s3-integration-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
tests:
name: S3 Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Build SeaweedFS
run: |
cd weed
go build -o weed -buildvcs=false
- name: Run S3 Integration Tests
timeout-minutes: 15
working-directory: test/s3/normal
run: |
set -x
echo "=== Running S3 Integration Tests ==="
go test -v -timeout=60s -run TestS3Integration ./...
- name: Run S3 DeleteBucketNotEmpty Tests
timeout-minutes: 15
working-directory: test/s3/normal
run: |
set -x
echo "=== Running S3 DeleteBucketNotEmpty Tests ==="
go test -v -timeout=60s -run TestS3DeleteBucketNotEmpty ./...
- name: Run S3 Empty Directory Marker Tests
timeout-minutes: 15
working-directory: test/s3/normal
run: |
set -x
echo "=== Running S3 Empty Directory Marker Tests ==="
go test -v -timeout=180s -run TestS3ListObjectsEmptyDirectoryMarkers ./...
- name: Run IAM Integration Tests
timeout-minutes: 15
working-directory: test/s3/normal
run: |
set -x
echo "=== Running IAM Integration Tests ==="
go test -v -timeout=60s -run TestIAMOperations ./...
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-test-logs
path: test/s3/normal/*.log
retention-days: 3
@@ -1,86 +0,0 @@
name: "S3 Filer Group Tests"
on:
pull_request:
concurrency:
group: ${{ github.head_ref }}/s3-filer-group-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
s3-filer-group-tests:
name: S3 Filer Group Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Filer Group Tests
timeout-minutes: 15
working-directory: test/s3/filer_group
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Filer Group Tests ==="
# Run tests with automatic server management
# The test-with-server target handles server startup/shutdown automatically
# Server is started with -filer.group=testgroup
make test-with-server || {
echo "❌ Filer group tests failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -100 weed-test.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Show server logs on failure
if: failure()
working-directory: test/s3/filer_group
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-filer-group-test-logs
path: test/s3/filer_group/weed-test*.log
retention-days: 3
-642
View File
@@ -1,642 +0,0 @@
name: "S3 Go Tests"
on:
pull_request:
concurrency:
group: ${{ github.head_ref }}/s3-go-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
s3-versioning-tests:
name: S3 Versioning Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
strategy:
matrix:
test-type: ["quick", "comprehensive"]
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Versioning Tests - ${{ matrix.test-type }}
timeout-minutes: 25
working-directory: test/s3/versioning
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
# The test-with-server target handles server startup/shutdown automatically
if [ "${{ matrix.test-type }}" = "quick" ]; then
# Override TEST_PATTERN for quick tests only
make test-with-server TEST_PATTERN="TestBucketListReturnDataVersioning|TestVersioningBasicWorkflow|TestVersioningDeleteMarkers"
else
# Run all versioning tests
make test-with-server
fi
- name: Show server logs on failure
if: failure()
working-directory: test/s3/versioning
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-versioning-test-logs-${{ matrix.test-type }}
path: test/s3/versioning/weed-test*.log
retention-days: 3
s3-versioning-compatibility:
name: S3 Versioning Compatibility Test
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run Core Versioning Test (Python s3tests equivalent)
timeout-minutes: 15
working-directory: test/s3/versioning
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run the specific test that is equivalent to the Python s3tests
make test-with-server || {
echo "❌ Test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -100 weed-test.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-versioning-compatibility-logs
path: test/s3/versioning/weed-test*.log
retention-days: 3
s3-cors-compatibility:
name: S3 CORS Compatibility Test
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run Core CORS Test (AWS S3 compatible)
timeout-minutes: 15
working-directory: test/s3/cors
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run the specific test that is equivalent to AWS S3 CORS behavior
make test-with-server || {
echo "❌ Test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -100 weed-test.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-cors-compatibility-logs
path: test/s3/cors/weed-test*.log
retention-days: 3
s3-retention-tests:
name: S3 Retention Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
strategy:
matrix:
test-type: ["quick", "comprehensive"]
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Retention Tests - ${{ matrix.test-type }}
timeout-minutes: 25
working-directory: test/s3/retention
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
# The test-with-server target handles server startup/shutdown automatically
if [ "${{ matrix.test-type }}" = "quick" ]; then
# Override TEST_PATTERN for quick tests only
make test-with-server TEST_PATTERN="TestBasicRetentionWorkflow|TestRetentionModeCompliance|TestLegalHoldWorkflow"
else
# Run all retention tests
make test-with-server
fi
- name: Show server logs on failure
if: failure()
working-directory: test/s3/retention
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-retention-test-logs-${{ matrix.test-type }}
path: test/s3/retention/weed-test*.log
retention-days: 3
s3-lifecycle-tests:
name: S3 Lifecycle Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Lifecycle Tests
timeout-minutes: 12
working-directory: test/s3/lifecycle
run: |
set -x
make test-with-server
- name: Show server logs on failure
if: failure()
working-directory: test/s3/lifecycle
run: |
if [ -f weed-test.log ]; then
echo "=== Last 200 lines of server logs ==="
tail -200 weed-test.log
fi
ps aux | grep -E "(weed|test)" || true
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-lifecycle-test-logs
path: test/s3/lifecycle/weed-test*.log
retention-days: 3
s3-checksum-tests:
name: S3 Checksum Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Checksum Tests
timeout-minutes: 16
working-directory: test/s3/checksum
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Tests ==="
make test-with-server
- name: Show server logs on failure
if: failure()
working-directory: test/s3/checksum
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-checksum-test-logs
path: test/s3/checksum/weed-test*.log
retention-days: 3
s3-cors-tests:
name: S3 CORS Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
strategy:
matrix:
test-type: ["quick", "comprehensive"]
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 CORS Tests - ${{ matrix.test-type }}
timeout-minutes: 25
working-directory: test/s3/cors
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
# The test-with-server target handles server startup/shutdown automatically
if [ "${{ matrix.test-type }}" = "quick" ]; then
# Override TEST_PATTERN for quick tests only
make test-with-server TEST_PATTERN="TestCORSConfigurationManagement|TestServiceLevelCORS|TestCORSBasicWorkflow"
else
# Run all CORS tests
make test-with-server
fi
- name: Show server logs on failure
if: failure()
working-directory: test/s3/cors
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-cors-test-logs-${{ matrix.test-type }}
path: test/s3/cors/weed-test*.log
retention-days: 3
s3-retention-worm:
name: S3 Retention WORM Integration Test
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run WORM Integration Tests
timeout-minutes: 15
working-directory: test/s3/retention
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run the WORM integration tests with automatic server management
# The test-with-server target handles server startup/shutdown automatically
make test-with-server TEST_PATTERN="TestWORM|TestRetentionExtendedAttributes|TestRetentionConcurrentOperations" || {
echo "❌ WORM integration test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -100 weed-test.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-retention-worm-logs
path: test/s3/retention/weed-test*.log
retention-days: 3
s3-versioning-stress:
name: S3 Versioning Stress Test
runs-on: ubuntu-22.04
timeout-minutes: 35
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Versioning Stress Tests
timeout-minutes: 30
working-directory: test/s3/versioning
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Start server for stress tests
make start-server
# Run stress tests (concurrent operations)
make test-versioning-stress || {
echo "❌ Stress test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -200 weed-test.log
fi
make stop-server
make clean
exit 1
}
# Run pagination stress tests (>1000 versions)
echo "=== Running pagination stress tests ==="
make test-versioning-pagination-stress || {
echo "❌ Pagination stress test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -200 weed-test.log
fi
make stop-server
make clean
exit 1
}
make stop-server
make clean
- name: Upload stress test logs
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-versioning-stress-logs
path: test/s3/versioning/weed-test*.log
retention-days: 7
s3-tagging-tests:
# CI job for S3 object tagging tests
name: S3 Tagging Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Tagging Tests
timeout-minutes: 15
working-directory: test/s3/tagging
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Set environment variables for the test
export S3_ENDPOINT="http://localhost:8006"
export S3_ACCESS_KEY="0555b35654ad1656d804"
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
# Run the specific test that is equivalent to AWS S3 tagging behavior
make test-with-server || {
echo "❌ Test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -100 weed-test.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-tagging-test-logs
path: test/s3/tagging/weed-test*.log
retention-days: 3
s3-remote-cache-tests:
name: S3 Remote Cache Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Remote Cache Tests
timeout-minutes: 15
working-directory: test/s3/remote_cache
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run the remote cache integration tests
# Tests singleflight deduplication for caching remote objects
make test-with-server || {
echo "❌ Test failed, checking logs..."
if [ -f primary-weed.log ]; then
echo "=== Primary server logs ==="
tail -100 primary-weed.log
fi
if [ -f remote-weed.log ]; then
echo "=== Remote server logs ==="
tail -100 remote-weed.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-remote-cache-test-logs
path: |
test/s3/remote_cache/primary-weed.log
test/s3/remote_cache/remote-weed.log
retention-days: 3
# Removed SSE-C integration tests and compatibility job
-309
View File
@@ -1,309 +0,0 @@
name: "S3 IAM Integration Tests"
on:
pull_request:
paths:
- 'weed/iam/**'
- 'weed/s3api/**'
- 'weed/credential/**'
- 'weed/pb/**'
- 'test/s3/iam/**'
- '.github/workflows/s3-iam-tests.yml'
push:
branches: [ master ]
paths:
- 'weed/iam/**'
- 'weed/s3api/**'
- 'weed/credential/**'
- 'weed/pb/**'
- 'test/s3/iam/**'
- '.github/workflows/s3-iam-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-iam-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
# Unit tests for IAM components
iam-unit-tests:
name: IAM Unit Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Get dependencies
run: |
go mod download
- name: Run IAM Unit Tests
timeout-minutes: 10
run: |
set -x
echo "=== Running IAM STS Tests ==="
go test -v -timeout 5m ./iam/sts/...
echo "=== Running IAM Policy Tests ==="
go test -v -timeout 5m ./iam/policy/...
echo "=== Running IAM Integration Tests ==="
go test -v -timeout 5m ./iam/integration/...
echo "=== Running S3 API IAM Tests ==="
go test -v -timeout 5m ./s3api/... -run ".*IAM.*|.*JWT.*|.*Auth.*"
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: iam-unit-test-results
path: |
weed/testdata/
weed/**/testdata/
retention-days: 3
# S3 IAM integration tests with SeaweedFS services
s3-iam-integration-tests:
name: S3 IAM Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 25
strategy:
matrix:
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
working-directory: weed
run: |
go install -buildvcs=false
- name: Run S3 IAM Integration Tests - ${{ matrix.test-type }}
timeout-minutes: 20
working-directory: test/s3/iam
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ==="
# Set WEED_BINARY to use the installed version
export WEED_BINARY=$(which weed)
export TEST_TIMEOUT=15m
# Run tests based on type
case "${{ matrix.test-type }}" in
"basic")
echo "Running basic IAM functionality tests..."
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./...
;;
"advanced")
echo "Running advanced IAM feature tests..."
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestS3IAMSessionExpiration|TestS3IAMMultipart|TestS3IAMPresigned" ./...
;;
"policy-enforcement")
echo "Running policy enforcement tests..."
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
;;
"group")
echo "Running IAM group management tests..."
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestIAMGroup" ./...
;;
"sts")
echo "Running STS and service account tests..."
make clean setup start-services wait-for-services
# SigV4-signed STS calls need admin credentials matching test_config.json.
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
export STS_TEST_ACCESS_KEY=test-access-key
export STS_TEST_SECRET_KEY=test-secret-key
# The use_service_account_credentials subtest is excluded because
# newly-created service-account access keys are not currently
# persisted to the filer after CreateServiceAccount — a
# pre-existing sync issue tracked separately from the
# GetFederationToken routing fix this PR addresses.
go test -v -timeout 15m \
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
./...
;;
*)
echo "Unknown test type: ${{ matrix.test-type }}"
exit 1
;;
esac
# Always cleanup
make stop-services
- name: Show service logs on failure
if: failure()
working-directory: test/s3/iam
run: |
echo "=== Service Logs ==="
echo "--- Master Log ---"
tail -50 weed-master.log 2>/dev/null || echo "No master log found"
echo ""
echo "--- Filer Log ---"
tail -50 weed-filer.log 2>/dev/null || echo "No filer log found"
echo ""
echo "--- Volume Log ---"
tail -50 weed-volume.log 2>/dev/null || echo "No volume log found"
echo ""
echo "--- S3 API Log ---"
tail -50 weed-s3.log 2>/dev/null || echo "No S3 log found"
echo ""
echo "=== Process Information ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-iam-integration-logs-${{ matrix.test-type }}
path: test/s3/iam/weed-*.log
retention-days: 5
# Distributed IAM tests
s3-iam-distributed-tests:
name: S3 IAM Distributed Tests
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
working-directory: weed
run: |
go install -buildvcs=false
- name: Run Distributed IAM Tests
timeout-minutes: 20
working-directory: test/s3/iam
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
export WEED_BINARY=$(which weed)
export TEST_TIMEOUT=15m
# Test distributed configuration
echo "Testing distributed IAM configuration..."
make clean setup
# Start services with distributed IAM config
echo "Starting services with distributed configuration..."
make start-services
make wait-for-services
# Run distributed-specific tests
export ENABLE_DISTRIBUTED_TESTS=true
go test -v -timeout 15m -run "TestS3IAMDistributedTests" ./... || {
echo "❌ Distributed tests failed, checking logs..."
make logs
exit 1
}
make stop-services
- name: Upload distributed test logs
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-iam-distributed-logs
path: test/s3/iam/weed-*.log
retention-days: 7
# Performance and stress tests
s3-iam-performance-tests:
name: S3 IAM Performance Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
working-directory: weed
run: |
go install -buildvcs=false
- name: Run IAM Performance Benchmarks
timeout-minutes: 25
working-directory: test/s3/iam
run: |
set -x
echo "=== Running IAM Performance Tests ==="
export WEED_BINARY=$(which weed)
export TEST_TIMEOUT=20m
make clean setup start-services wait-for-services
# Run performance tests (benchmarks disabled for CI)
echo "Running performance tests..."
export ENABLE_PERFORMANCE_TESTS=true
go test -v -timeout 15m -run "TestS3IAMPerformanceTests" ./... || {
echo "❌ Performance tests failed"
make logs
exit 1
}
make stop-services
- name: Upload performance test results
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-iam-performance-results
path: |
test/s3/iam/weed-*.log
test/s3/iam/*.test
retention-days: 7
-161
View File
@@ -1,161 +0,0 @@
name: "S3 Keycloak Integration Tests"
on:
pull_request:
paths:
- 'weed/iam/**'
- 'weed/s3api/**'
- 'test/s3/iam/**'
- '.github/workflows/s3-keycloak-tests.yml'
push:
branches: [ master ]
paths:
- 'weed/iam/**'
- 'weed/s3api/**'
- 'test/s3/iam/**'
- '.github/workflows/s3-keycloak-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-keycloak-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
# Dedicated job for Keycloak integration tests
s3-keycloak-integration-tests:
name: S3 Keycloak Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
working-directory: weed
run: |
go install -buildvcs=false
- name: Run Keycloak Integration Tests
timeout-minutes: 25
working-directory: test/s3/iam
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting S3 Keycloak Integration Tests ==="
# Set WEED_BINARY to use the installed version
export WEED_BINARY=$(which weed)
export TEST_TIMEOUT=20m
echo "Running Keycloak integration tests..."
# Start Keycloak container first
docker run -d \
--name keycloak \
-p 8080:8080 \
-e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
-e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \
-e KC_HTTP_ENABLED=true \
-e KC_HOSTNAME_STRICT=false \
-e KC_HOSTNAME_STRICT_HTTPS=false \
quay.io/keycloak/keycloak:26.0 \
start-dev
# Wait for Keycloak with better health checking
timeout 300 bash -c '
while true; do
if curl -s http://localhost:8080/health/ready > /dev/null 2>&1; then
echo "✅ Keycloak health check passed"
break
fi
echo "... waiting for Keycloak to be ready"
sleep 5
done
'
# Setup Keycloak configuration
./setup_keycloak.sh
# Start SeaweedFS services
make clean setup start-services wait-for-services
# Verify service accessibility
echo "=== Verifying Service Accessibility ==="
curl -f http://localhost:8080/realms/master
curl -s http://localhost:8333/healthz
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
# Run Keycloak-specific tests
echo "=== Running Keycloak Tests ==="
export KEYCLOAK_URL=http://localhost:8080
export S3_ENDPOINT=http://localhost:8333
# Wait for realm to be properly configured
timeout 120 bash -c 'until curl -fs http://localhost:8080/realms/seaweedfs-test/.well-known/openid-configuration > /dev/null; do echo "... waiting for realm"; sleep 3; done'
# Run the Keycloak integration tests
go test -v -timeout 20m -run "TestKeycloak" ./...
- name: Show server logs on failure
if: failure()
working-directory: test/s3/iam
run: |
echo "=== Service Logs ==="
echo "--- Keycloak logs ---"
docker logs keycloak --tail=100 || echo "No Keycloak container logs"
echo "--- SeaweedFS Master logs ---"
if [ -f weed-master.log ]; then
tail -100 weed-master.log
fi
echo "--- SeaweedFS S3 logs ---"
if [ -f weed-s3.log ]; then
tail -100 weed-s3.log
fi
echo "--- SeaweedFS Filer logs ---"
if [ -f weed-filer.log ]; then
tail -100 weed-filer.log
fi
echo "=== System Status ==="
ps aux | grep -E "(weed|keycloak)" || true
netstat -tlnp | grep -E "(8333|9333|8080|8888)" || true
docker ps -a || true
- name: Cleanup
if: always()
working-directory: test/s3/iam
run: |
# Stop Keycloak container
docker stop keycloak || true
docker rm keycloak || true
# Stop SeaweedFS services
make clean || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-keycloak-test-logs
path: |
test/s3/iam/*.log
test/s3/iam/test-volume-data/
retention-days: 3
@@ -1,133 +0,0 @@
name: "S3 Mutation Regression Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'test/s3/delete/**'
- 'test/s3/distributed_lock/**'
- 'test/s3/versioning/**'
- 'test/volume_server/framework/**'
- 'docker/compose/s3.json'
- '.github/workflows/s3-mutation-regression-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-mutation-regression-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-versioning-regressions:
name: S3 Versioning Regression Tests
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Run S3 versioning regression tests
timeout-minutes: 20
working-directory: test/s3/versioning
run: |
set -x
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
- name: Show server logs on failure
if: failure()
working-directory: test/s3/versioning
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
tail -100 weed-test.log
fi
- name: Upload versioning logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-versioning-regression-logs
path: test/s3/versioning/weed-test*.log
retention-days: 3
s3-delete-regressions:
name: S3 Delete Regression Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Run S3 delete regression tests
timeout-minutes: 15
working-directory: test/s3/delete
run: |
set -x
make test-with-server
- name: Show server logs on failure
if: failure()
working-directory: test/s3/delete
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
tail -100 weed-test.log
fi
- name: Upload delete logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-delete-regression-logs
path: test/s3/delete/weed-test*.log
retention-days: 3
s3-distributed-lock-regressions:
name: S3 Distributed Lock Regression Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS
run: |
go build -o weed/weed -buildvcs=false ./weed
- name: Run distributed lock regressions
timeout-minutes: 25
env:
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: |
set -x
mkdir -p "$TMPDIR"
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
- name: Upload distributed lock logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-distributed-lock-regression-logs
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
retention-days: 3
-163
View File
@@ -1,163 +0,0 @@
name: "S3 PyArrow Parquet Tests"
on:
push:
branches: [master]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'test/s3/parquet/**'
- '.github/workflows/s3-parquet-tests.yml'
pull_request:
branches: [master]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'test/s3/parquet/**'
- '.github/workflows/s3-parquet-tests.yml'
workflow_dispatch:
env:
S3_ACCESS_KEY: some_access_key1
S3_SECRET_KEY: some_secret_key1
S3_ENDPOINT_URL: http://localhost:8333
BUCKET_NAME: test-parquet-bucket
jobs:
parquet-integration-tests:
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
python-version: ['3.9', '3.11', '3.12']
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'test/s3/parquet/requirements.txt'
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y lsof netcat-openbsd
- name: Build SeaweedFS
run: |
cd weed
go build -v
sudo cp weed /usr/local/bin/
weed version
- name: Run PyArrow Parquet integration tests
run: |
cd test/s3/parquet
make test-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
VOLUME_MAX_SIZE_MB: 50
- name: Run implicit directory fix tests
run: |
cd test/s3/parquet
make test-implicit-dir-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run PyArrow native S3 filesystem tests
run: |
cd test/s3/parquet
make test-native-s3-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run cross-filesystem compatibility tests
run: |
cd test/s3/parquet
TEST_QUICK=1 make test-cross-fs-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run SSE-S3 encryption compatibility tests
run: |
cd test/s3/parquet
make test-sse-s3-compat
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: test-logs-python-${{ matrix.python-version }}
path: |
/tmp/seaweedfs-parquet-*.log
test/s3/parquet/*.log
retention-days: 7
- name: Cleanup
if: always()
run: |
cd test/s3/parquet
make stop-seaweedfs-safe || true
make clean || true
unit-tests:
name: Go Unit Tests (Implicit Directory)
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: ^1.25
cache: true
- name: Run Go unit tests
run: |
cd weed/s3api
go test -v -run TestImplicitDirectory
- name: Run all S3 API tests
run: |
cd weed/s3api
go test -v -timeout 5m
-426
View File
@@ -1,426 +0,0 @@
name: "S3 Policy Integration Tests"
on:
pull_request:
paths:
- 'weed/s3api/s3_iam_middleware.go'
- 'weed/s3api/s3api_bucket_policy*.go'
- 'weed/s3api/s3_action_resolver.go'
- 'weed/s3api/policy/**'
- 'weed/iam/**'
- 'test/s3/iam/**'
- 'test/s3/policy/**'
- '.github/workflows/s3-policy-tests.yml'
push:
branches: [ master, main ]
paths:
- 'weed/s3api/s3_iam_middleware.go'
- 'weed/s3api/s3api_bucket_policy*.go'
- 'weed/s3api/s3_action_resolver.go'
- 'weed/s3api/policy/**'
- 'weed/iam/**'
- 'test/s3/iam/**'
- 'test/s3/policy/**'
- '.github/workflows/s3-policy-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-policy-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
# Unit tests for policy components
policy-unit-tests:
name: S3 Policy Unit Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Get dependencies
run: |
go mod download
- name: Run S3 Policy Unit Tests
timeout-minutes: 10
run: |
set -x
echo "=== Running S3 Action Resolver Tests ==="
go test -v -timeout 5m ./s3api/... -run ".*ActionResolver.*"
echo "=== Running S3 Bucket Policy Engine Tests ==="
go test -v -timeout 5m ./s3api/... -run ".*BucketPolicy.*|.*PolicyEngine.*"
echo "=== Running IAM Policy Tests ==="
go test -v -timeout 5m ./iam/policy/...
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: policy-unit-test-results
path: |
weed/testdata/
weed/**/testdata/
retention-days: 3
# S3 Policy Variables Integration Tests
s3-policy-variables-tests:
name: S3 Policy Variables Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Policy Variables Integration Tests
timeout-minutes: 20
working-directory: test/s3/iam
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting S3 Policy Variables Integration Tests ==="
# Set WEED_BINARY to use the installed version
export WEED_BINARY=$(which weed)
export TEST_TIMEOUT=15m
# Run policy variables tests
echo "Running policy variables tests..."
# Kill any existing weed server on port 8333
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
kill $(lsof -t -i:8333) 2>/dev/null || true
sleep 2
fi
# Start weed server with IAM configuration
echo "Starting weed server with IAM configuration..."
$WEED_BINARY server \
-s3 \
-s3.port=8333 \
-s3.iam.config="$(pwd)/test_iam_config.json" \
-filer \
-volume.max=0 \
-master.volumeSizeLimitMB=100 \
-s3.allowDeleteBucketNotEmpty=true \
> /tmp/weed_policy_test_server.log 2>&1 &
SERVER_PID=$!
echo "Server started with PID: $SERVER_PID"
# Wait for server to be ready
echo "Waiting for server to be ready..."
MAX_WAIT=30
COUNTER=0
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
sleep 1
COUNTER=$((COUNTER + 1))
if [ $COUNTER -ge $MAX_WAIT ]; then
echo "Server failed to start within ${MAX_WAIT} seconds"
echo "Server log:"
cat /tmp/weed_policy_test_server.log
kill $SERVER_PID 2>/dev/null || true
exit 1
fi
done
echo "Server is ready!"
# Trap to ensure server is killed on exit
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
# Run the tests
go test -v -timeout 15m -run TestS3PolicyVariables ./...
- name: Show service logs on failure
if: failure()
working-directory: test/s3/iam
run: |
echo "=== Service Logs ==="
if [ -f /tmp/weed_policy_test_server.log ]; then
echo "--- Last 100 lines of Server Log ---"
tail -100 /tmp/weed_policy_test_server.log
fi
echo ""
echo "=== Process Information ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-policy-variables-test-logs
path: /tmp/weed_policy_test_server.log
retention-days: 5
# S3 Policy Enforcement Integration Tests
s3-policy-enforcement-tests:
name: S3 Policy Enforcement Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
strategy:
matrix:
test-case: ["basic-policy", "contextual-policy", "advanced-policy"]
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Policy Enforcement Tests - ${{ matrix.test-case }}
timeout-minutes: 25
working-directory: test/s3/iam
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
export WEED_BINARY=$(which weed)
export TEST_TIMEOUT=20m
# Kill any existing weed server on port 8333
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
kill $(lsof -t -i:8333) 2>/dev/null || true
sleep 2
fi
# Start weed server with IAM configuration
echo "Starting weed server with IAM configuration..."
$WEED_BINARY server \
-s3 \
-s3.port=8333 \
-s3.iam.config="$(pwd)/test_iam_config.json" \
-filer \
-volume.max=0 \
-master.volumeSizeLimitMB=100 \
-s3.allowDeleteBucketNotEmpty=true \
> /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log 2>&1 &
SERVER_PID=$!
echo "Server started with PID: $SERVER_PID"
# Wait for server to be ready
echo "Waiting for server to be ready..."
MAX_WAIT=30
COUNTER=0
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
sleep 1
COUNTER=$((COUNTER + 1))
if [ $COUNTER -ge $MAX_WAIT ]; then
echo "Server failed to start within ${MAX_WAIT} seconds"
cat /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
kill $SERVER_PID 2>/dev/null || true
exit 1
fi
done
echo "Server is ready!"
# Trap to ensure server is killed on exit
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
# Run tests based on test case
case "${{ matrix.test-case }}" in
"basic-policy")
echo "Running basic policy enforcement tests..."
go test -v -timeout 20m -run "TestS3IAMBucketPolicy|TestS3IAMPolicyEnforcement" ./...
;;
"contextual-policy")
echo "Running contextual policy tests..."
go test -v -timeout 20m -run "TestS3PolicyVariables|TestS3IAMContextual" ./...
;;
"advanced-policy")
echo "Running advanced policy tests..."
go test -v -timeout 20m -run "TestS3IAMMultipart|TestS3IAMPresigned" ./...
;;
*)
echo "Unknown test case: ${{ matrix.test-case }}"
exit 1
;;
esac
- name: Show service logs on failure
if: failure()
working-directory: test/s3/iam
run: |
echo "=== Service Logs ==="
if [ -f /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log ]; then
echo "--- Last 100 lines of Server Log ---"
tail -100 /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
fi
echo ""
echo "=== Process Information ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
retention-days: 5
# Trusted Proxy Detection Tests
trusted-proxy-tests:
name: Trusted Proxy Detection Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run Trusted Proxy Tests
timeout-minutes: 15
working-directory: test/s3/iam
run: |
set -x
echo "=== Running Trusted Proxy Detection Tests ==="
export WEED_BINARY=$(which weed)
# Kill any existing weed server on port 8333
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
kill $(lsof -t -i:8333) 2>/dev/null || true
sleep 2
fi
# Start weed server
echo "Starting weed server..."
$WEED_BINARY server \
-s3 \
-s3.port=8333 \
-s3.iam.config="$(pwd)/test_iam_config.json" \
-filer \
-volume.max=0 \
-master.volumeSizeLimitMB=100 \
-s3.allowDeleteBucketNotEmpty=true \
> /tmp/weed_proxy_test.log 2>&1 &
SERVER_PID=$!
echo "Server started with PID: $SERVER_PID"
# Wait for server to be ready
echo "Waiting for server to be ready..."
MAX_WAIT=30
COUNTER=0
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
sleep 1
COUNTER=$((COUNTER + 1))
if [ $COUNTER -ge $MAX_WAIT ]; then
echo "Server failed to start within ${MAX_WAIT} seconds"
kill $SERVER_PID 2>/dev/null || true
exit 1
fi
done
# Trap to ensure server is killed on exit
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
# Run proxy tests
go test -v -timeout 10m -run "TestTrustedProxy|TestPrivateIP" ./...
- name: Show service logs on failure
if: failure()
run: |
echo "=== Service Logs ==="
if [ -f /tmp/weed_proxy_test.log ]; then
echo "--- Last 100 lines of Server Log ---"
tail -100 /tmp/weed_proxy_test.log
fi
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: trusted-proxy-test-logs
path: /tmp/weed_proxy_test.log
retention-days: 3
# S3 Policy Shell Integration Tests
s3-policy-shell-tests:
name: S3 Policy Shell Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Policy Shell Tests
timeout-minutes: 10
working-directory: test/s3/policy
run: |
set -x
echo "=== Running S3 Policy Shell Tests ==="
# Set WEED_BINARY to use the installed version (though test uses 'weed' command)
export WEED_BINARY=$(which weed)
export PATH=$PATH:$(dirname $WEED_BINARY)
go test -v -timeout 10m ./...
@@ -1,122 +0,0 @@
name: "S3 Proxy Signature Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
proxy-signature-tests:
name: S3 Proxy Signature Verification Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Build SeaweedFS binary for Linux
run: |
set -x
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -v -o test/s3/proxy_signature/weed ./weed
- name: Run S3 Proxy Signature Tests
timeout-minutes: 10
working-directory: test/s3/proxy_signature
run: |
set -x
echo "Starting Docker Compose services..."
docker compose up -d --build
# Check if containers are running
echo "Checking container status..."
docker compose ps
# Wait for services to be ready
echo "Waiting for nginx proxy to be ready..."
PROXY_READY=0
for i in $(seq 1 30); do
if curl -s http://localhost:9000/ > /dev/null 2>&1; then
echo "Proxy is ready"
PROXY_READY=1
break
fi
echo "Waiting for proxy... ($i/30)"
sleep 1
done
if [ $PROXY_READY -eq 0 ]; then
echo "ERROR: Proxy failed to become ready after 30 seconds"
echo "Docker compose logs:"
docker compose logs --no-color || true
exit 1
fi
# Wait for SeaweedFS to be ready
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
S3_READY=0
for i in $(seq 1 30); do
# Check logs first for startup message (weed mini says "S3 service is ready")
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)"; then
echo "SeaweedFS S3 gateway is ready"
S3_READY=1
break
fi
# Fallback: check headers via proxy (which is already ready)
if curl -s -I http://localhost:9000/ | grep -qi "SeaweedFS"; then
echo "SeaweedFS S3 gateway is responding via proxy"
S3_READY=1
break
fi
echo "Waiting for S3 gateway... ($i/30)"
sleep 1
done
if [ $S3_READY -eq 0 ]; then
echo "ERROR: SeaweedFS S3 gateway failed to become ready after 30 seconds"
echo "Latest seaweedfs logs:"
docker compose logs --no-color --tail 20 seaweedfs || true
exit 1
fi
# Run the test script inside AWS CLI container
echo "Running test script..."
docker run --rm --network host \
--entrypoint bash \
amazon/aws-cli:latest \
-c "$(cat test.sh)"
TEST_RESULT=$?
# Cleanup
docker compose down
exit $TEST_RESULT
- name: Cleanup on failure
if: failure()
working-directory: test/s3/proxy_signature
run: |
echo "Cleaning up Docker containers..."
ls -al weed || true
ldd weed || true
echo "Docker compose logs:"
docker compose logs --no-color || true
echo "Container status before cleanup:"
docker ps -a
echo "Stopping services..."
docker compose down || true
-81
View File
@@ -1,81 +0,0 @@
name: "S3 Spark Integration Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'test/s3/spark/**'
- 'test/s3tables/testutil/**'
- '.github/workflows/s3-spark-tests.yml'
workflow_dispatch:
concurrency:
group: ${{ github.head_ref }}/s3-spark-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-spark-issue-repro-tests:
name: S3 Spark Issue Reproduction Tests
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Pre-pull Spark image
run: docker pull apache/spark:3.5.8
- name: Run S3 Spark integration tests
working-directory: test/s3/spark
timeout-minutes: 35
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting S3 Spark Integration Tests ==="
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
echo "S3 Spark integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3/spark
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|spark)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-spark-test-logs
path: test/s3/spark/test-output.log
retention-days: 3
-401
View File
@@ -1,401 +0,0 @@
name: "S3 SSE Tests"
on:
pull_request:
paths:
- 'weed/s3api/s3_sse_*.go'
- 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'weed/s3api/s3api_object_handlers_copy*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'weed/kms/**'
- 'test/s3/sse/**'
- '.github/workflows/s3-sse-tests.yml'
push:
branches: [ master, main ]
paths:
- 'weed/s3api/s3_sse_*.go'
- 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'weed/s3api/s3api_object_handlers_copy*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'weed/kms/**'
- 'test/s3/sse/**'
concurrency:
group: ${{ github.head_ref }}/s3-sse-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
s3-sse-integration-tests:
name: S3 SSE Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
strategy:
matrix:
test-type: ["quick", "comprehensive"]
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 SSE Integration Tests - ${{ matrix.test-type }}
timeout-minutes: 25
working-directory: test/s3/sse
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting SSE Tests ==="
# Run tests with automatic server management
# The test-with-server target handles server startup/shutdown automatically
if [ "${{ matrix.test-type }}" = "quick" ]; then
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
else
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
# same change that fixed them.
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
fi
- name: Show server logs on failure
if: failure()
working-directory: test/s3/sse
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080|8888)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-sse-test-logs-${{ matrix.test-type }}
path: test/s3/sse/weed-test*.log
retention-days: 3
s3-sse-compatibility:
name: S3 SSE Compatibility Test
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run Core SSE Compatibility Test (AWS S3 equivalent)
timeout-minutes: 15
working-directory: test/s3/sse
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality plus Range requests
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|.*RangeRequestsServerBehavior" || {
echo "❌ SSE compatibility test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -100 weed-test.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-sse-compatibility-logs
path: test/s3/sse/weed-test*.log
retention-days: 3
s3-sse-metadata-persistence:
name: S3 SSE Metadata Persistence Test
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run SSE Metadata Persistence Test
timeout-minutes: 15
working-directory: test/s3/sse
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run the specific test that would catch filer metadata storage bugs
# This test validates that encryption metadata survives the full PUT/GET cycle
make test-metadata-persistence || {
echo "❌ SSE metadata persistence test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -100 weed-test.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-sse-metadata-persistence-logs
path: test/s3/sse/weed-test*.log
retention-days: 3
s3-sse-copy-operations:
name: S3 SSE Copy Operations Test
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run SSE Copy Operations Tests
timeout-minutes: 20
working-directory: test/s3/sse
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run tests that validate SSE copy operations and cross-encryption scenarios
echo "🚀 Running SSE copy operations tests..."
echo "📋 Note: SSE-C copy operations have pre-existing functionality gaps"
echo " Cross-encryption copy security fix has been implemented and maintained"
# Skip SSE-C copy operations due to pre-existing HTTP 500 errors
# The critical security fix for cross-encryption (SSE-C → SSE-KMS) has been preserved
echo "⏭️ Skipping SSE copy operations tests due to known limitations:"
echo " - SSE-C copy operations: HTTP 500 errors (pre-existing functionality gap)"
echo " - Cross-encryption security fix: ✅ Implemented and tested (forces streaming copy)"
echo " - These limitations are documented as pre-existing issues"
exit 0 # Job succeeds with security fix preserved and limitations documented
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-sse-copy-operations-logs
path: test/s3/sse/weed-test*.log
retention-days: 3
s3-sse-multipart:
name: S3 SSE Multipart Upload Test
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run SSE Multipart Upload Tests
timeout-minutes: 20
working-directory: test/s3/sse
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Multipart tests - Document known architectural limitations
echo "🚀 Running multipart upload tests..."
echo "📋 Note: SSE-KMS multipart upload has known architectural limitation requiring per-chunk metadata storage"
echo " SSE-C multipart tests will be skipped due to pre-existing functionality gaps"
# Test SSE-C basic multipart (skip advanced multipart that fails with HTTP 500)
# Skip SSE-KMS multipart due to architectural limitation (each chunk needs independent metadata)
echo "⏭️ Skipping multipart upload tests due to known limitations:"
echo " - SSE-C multipart GET operations: HTTP 500 errors (pre-existing functionality gap)"
echo " - SSE-KMS multipart decryption: Requires per-chunk SSE metadata architecture changes"
echo " - These limitations are documented and require future architectural work"
exit 0 # Job succeeds with clear documentation of known limitations
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-sse-multipart-logs
path: test/s3/sse/weed-test*.log
retention-days: 3
s3-sse-performance:
name: S3 SSE Performance Test
runs-on: ubuntu-22.04
timeout-minutes: 35
# Only run performance tests on master branch pushes to avoid overloading PR testing
if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 SSE Performance Tests
timeout-minutes: 30
working-directory: test/s3/sse
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run performance tests with various data sizes
make perf || {
echo "❌ SSE performance test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
tail -200 weed-test.log
fi
make clean
exit 1
}
make clean
- name: Upload performance test logs
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-sse-performance-logs
path: test/s3/sse/weed-test*.log
retention-days: 7
s3-volume-encryption:
name: S3 Volume Encryption Test
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Volume Encryption Integration Tests
timeout-minutes: 15
working-directory: test/s3/sse
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run volume encryption tests with -s3.encryptVolumeData flag
echo "🚀 Running S3 volume encryption integration tests..."
make test-volume-encryption || {
echo "❌ Volume encryption tests failed, checking logs..."
if [ -f /tmp/seaweedfs-sse-mini.log ]; then
echo "=== Server logs ==="
tail -100 /tmp/seaweedfs-sse-mini.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-volume-encryption-logs
path: /tmp/seaweedfs-sse-*.log
retention-days: 3
-895
View File
@@ -1,895 +0,0 @@
name: "S3 Tables Integration Tests"
on:
pull_request:
permissions:
contents: read
jobs:
s3-tables-tests:
name: S3 Tables Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run S3 Tables Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/table-buckets
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting S3 Tables Tests ==="
# Run S3 Tables integration tests
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "S3 Tables integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/table-buckets
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-tables-test-logs
path: test/s3tables/table-buckets/test-output.log
retention-days: 3
iceberg-catalog-tests:
name: Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Run go mod tidy
run: go mod tidy
- name: Run Iceberg Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/catalog
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Iceberg Catalog Tests ==="
# Run Iceberg catalog integration tests using Makefile (handles build)
make test 2>&1 | tee test-output.log || {
echo "Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: iceberg-catalog-test-logs
path: test/s3tables/catalog/test-output.log
retention-days: 3
trino-iceberg-catalog-tests:
name: Trino Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Trino image
run: docker pull trinodb/trino:479
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Trino Iceberg Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/catalog_trino
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Trino Iceberg Catalog Tests ==="
# Run Trino + Iceberg catalog integration tests
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Trino Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_trino
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: trino-iceberg-catalog-test-logs
path: test/s3tables/catalog_trino/test-output.log
retention-days: 3
dremio-iceberg-catalog-tests:
name: Dremio Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Dremio image
run: docker pull dremio/dremio-oss:25.2.0
- name: Pre-pull Python image for PyIceberg writer
run: docker pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Dremio Iceberg Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/catalog_dremio
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Dremio Iceberg Catalog Tests ==="
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Dremio Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_dremio
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|dremio)" || true
echo "=== Dremio containers ==="
docker ps -a --filter "name=seaweed-dremio" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: dremio-iceberg-catalog-test-logs
path: test/s3tables/catalog_dremio/test-output.log
retention-days: 3
doris-iceberg-catalog-tests:
name: Doris Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 35
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Doris image
run: docker pull apache/doris:doris-all-in-one-2.1.0
- name: Pre-pull Python image for PyIceberg writer
run: docker pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Doris Iceberg Catalog Integration Tests
timeout-minutes: 30
working-directory: test/s3tables/catalog_doris
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Doris Iceberg Catalog Tests ==="
go test -v -timeout 25m . 2>&1 | tee test-output.log || {
echo "Doris Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_doris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|doris)" || true
echo "=== Doris containers ==="
docker ps -a --filter "name=seaweed-doris" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: doris-iceberg-catalog-test-logs
path: test/s3tables/catalog_doris/test-output.log
retention-days: 3
polaris-integration-tests:
name: Polaris Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Pre-pull Polaris image
run: docker pull apache/polaris:latest
- name: Run Polaris Integration Tests
timeout-minutes: 25
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Polaris Tests ==="
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
echo "Polaris integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/polaris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: polaris-test-logs
path: test/s3tables/polaris/test-output.log
retention-days: 3
spark-iceberg-catalog-tests:
name: Spark Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Spark image
run: docker pull apache/spark:3.5.1
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Spark Iceberg Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/catalog_spark
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Spark Iceberg Catalog Tests ==="
# Run Spark + Iceberg catalog integration tests
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Spark Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_spark
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: spark-iceberg-catalog-test-logs
path: test/s3tables/catalog_spark/test-output.log
retention-days: 3
risingwave-catalog-tests:
name: RisingWave Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull RisingWave image
run: |
docker pull risingwavelabs/risingwave:v2.5.0
docker pull postgres:16-alpine
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run RisingWave Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/catalog_risingwave
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting RisingWave Catalog Tests ==="
# Run RisingWave catalog integration tests
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "RisingWave catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_risingwave
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: risingwave-catalog-test-logs
path: test/s3tables/catalog_risingwave/test-output.log
retention-days: 3
sts-integration-tests:
name: STS Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Python image
run: docker pull python:3
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run STS Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/sts_integration
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting STS Integration Tests ==="
# Run STS integration tests
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "STS integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/sts_integration
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: sts-integration-test-logs
path: test/s3tables/sts_integration/test-output.log
retention-days: 3
lakekeeper-integration-tests:
name: Lakekeeper Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Python image
run: docker pull python:3
- name: Pre-pull LocalStack image (if needed)
run: docker pull localstack/localstack:latest || true
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Lakekeeper Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/lakekeeper
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Lakekeeper Integration Tests ==="
# Run Lakekeeper integration tests
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Lakekeeper integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/lakekeeper
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: lakekeeper-integration-test-logs
path: test/s3tables/lakekeeper/test-output.log
retention-days: 3
unity-catalog-integration-tests:
name: Unity Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Unity Catalog image
run: docker pull unitycatalog/unitycatalog:v0.4.0
- name: Pre-pull Python image for delta-rs writer
run: docker pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Unity Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/unity_catalog
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Unity Catalog Tests ==="
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Unity Catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/unity_catalog
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
echo "=== Unity Catalog containers ==="
docker ps -a --filter "name=seaweed-unity-catalog" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: unity-catalog-integration-test-logs
path: test/s3tables/unity_catalog/test-output.log
retention-days: 3
s3-tables-build-verification:
name: S3 Tables Build Verification
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Verify S3 Tables Package Builds
run: |
set -x
echo "=== Building S3 Tables package ==="
go build ./weed/s3api/s3tables || {
echo "S3 Tables package build failed"
exit 1
}
echo "S3 Tables package built successfully"
- name: Verify S3 API Integration Builds
run: |
set -x
echo "=== Building S3 API with S3 Tables integration ==="
go build ./weed/s3api || {
echo "S3 API build with S3 Tables failed"
exit 1
}
echo "S3 API with S3 Tables integration built successfully"
- name: Run Go Tests for S3 Tables Package
run: |
set -x
echo "=== Running Go unit tests for S3 Tables ==="
go test -v -race -timeout 5m ./weed/s3api/s3tables/... || {
echo "S3 Tables unit tests failed"
exit 1
}
echo "S3 Tables unit tests passed"
- name: Verify Iceberg Package Builds
run: |
set -x
echo "=== Building Iceberg package ==="
go build ./weed/s3api/iceberg || {
echo "Iceberg package build failed"
exit 1
}
echo "Iceberg package built successfully"
- name: Run Go Tests for Iceberg Package
run: |
set -x
echo "=== Running Go unit tests for Iceberg ==="
go test -v -race -timeout 5m ./weed/s3api/iceberg/... || {
echo "Iceberg unit tests failed"
exit 1
}
echo "Iceberg unit tests passed"
s3-tables-fmt-check:
name: S3 Tables Format Check
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Check Go Format
run: |
set -x
echo "=== Checking S3 Tables Go format ==="
unformatted=$(gofmt -l ./weed/s3api/s3tables)
if [ -n "$unformatted" ]; then
echo "Go format check failed - files need formatting"
echo "$unformatted"
exit 1
fi
echo "All S3 Tables files are properly formatted"
- name: Check S3 Tables Test Format
run: |
set -x
echo "=== Checking S3 Tables test format ==="
unformatted=$(gofmt -l ./test/s3tables)
if [ -n "$unformatted" ]; then
echo "Go format check failed for tests"
echo "$unformatted"
exit 1
fi
echo "All S3 Tables test files are properly formatted"
- name: Check Iceberg Format
run: |
set -x
echo "=== Checking Iceberg Go format ==="
unformatted=$(gofmt -l ./weed/s3api/iceberg)
if [ -n "$unformatted" ]; then
echo "Go format check failed for Iceberg - files need formatting"
echo "$unformatted"
exit 1
fi
echo "All Iceberg files are properly formatted"
s3-tables-vet:
name: S3 Tables Go Vet Check
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Run Go Vet
run: |
set -x
echo "=== Running go vet on S3 Tables package ==="
go vet ./weed/s3api/s3tables/... || {
echo "go vet check failed"
exit 1
}
echo "go vet checks passed"
- name: Run Go Vet on Tests
run: |
set -x
echo "=== Running go vet on S3 Tables tests ==="
go vet ./test/s3tables/... || {
echo "go vet check failed for tests"
exit 1
}
echo "go vet checks passed for tests"
- name: Run Go Vet on Iceberg
run: |
set -x
echo "=== Running go vet on Iceberg package ==="
go vet ./weed/s3api/iceberg/... || {
echo "go vet check failed for Iceberg"
exit 1
}
echo "go vet checks passed for Iceberg"
File diff suppressed because it is too large Load Diff
-93
View File
@@ -1,93 +0,0 @@
name: "SFTP Integration Tests"
on:
push:
branches: [ master, main ]
paths:
- 'weed/sftpd/**'
- 'weed/command/sftp.go'
- 'test/sftp/**'
- '.github/workflows/sftp-tests.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/sftpd/**'
- 'weed/command/sftp.go'
- 'test/sftp/**'
- '.github/workflows/sftp-tests.yml'
concurrency:
group: ${{ github.head_ref }}/sftp-tests
cancel-in-progress: true
permissions:
contents: read
env:
TEST_TIMEOUT: '15m'
jobs:
sftp-integration:
name: SFTP Integration Testing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y openssh-client
- name: Build SeaweedFS
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Run SFTP Integration Tests
run: |
cd test/sftp
echo "🧪 Running SFTP integration tests..."
echo "============================================"
# Install test dependencies
go mod download
# Run all SFTP tests
go test -v -timeout=${{ env.TEST_TIMEOUT }} ./...
echo "============================================"
echo "✅ SFTP integration tests completed"
- name: Test Summary
if: always()
run: |
echo "## 🔐 SFTP Integration Test Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **HomeDir Path Translation**: User home directory mapping (fixes #7470)" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **File Operations**: Upload, download, delete" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Directory Operations**: Create, list, remove" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Large File Handling**: 1MB+ file support" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Path Edge Cases**: Unicode, trailing slashes, .. paths" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Admin Access**: Root user verification" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Test Configuration" >> $GITHUB_STEP_SUMMARY
echo "| User | HomeDir | Permissions |" >> $GITHUB_STEP_SUMMARY
echo "|------|---------|-------------|" >> $GITHUB_STEP_SUMMARY
echo "| admin | / | Full access |" >> $GITHUB_STEP_SUMMARY
echo "| testuser | /sftp/testuser | Home directory only |" >> $GITHUB_STEP_SUMMARY
echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY
@@ -1,263 +0,0 @@
name: Spark Integration Tests
on:
push:
paths:
- 'test/java/spark/**'
- 'other/java/hdfs3/**'
- 'other/java/client/**'
- '.github/workflows/spark-integration-tests.yml'
pull_request:
paths:
- 'test/java/spark/**'
- 'other/java/hdfs3/**'
- 'other/java/client/**'
- '.github/workflows/spark-integration-tests.yml'
workflow_dispatch:
permissions:
contents: read
checks: write
pull-requests: write
jobs:
spark-integration-tests:
name: Spark Integration Tests
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
# ========================================
# SETUP & BUILD
# ========================================
- name: Checkout code
uses: actions/checkout@v6
- name: Set up JDK 11
uses: actions/setup-java@v5
with:
java-version: '11'
distribution: 'temurin'
cache: maven
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS binary
run: |
echo "Building SeaweedFS binary (statically linked for Alpine)..."
cd weed
CGO_ENABLED=0 go build -o ../docker/weed
cd ../docker
ls -la weed filer.toml entrypoint.sh
file weed
echo "OK SeaweedFS binary built"
- name: Build SeaweedFS Java dependencies
run: |
echo "Building Java client..."
cd other/java/client
mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
echo "OK Java client built"
cd ../../..
echo "Building HDFS3 client..."
cd other/java/hdfs3
mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
echo "OK HDFS3 client built"
echo ""
echo "All Java dependencies installed to ~/.m2/repository"
# ========================================
# SPARK INTEGRATION TESTS (DOCKER)
# ========================================
- name: Start SeaweedFS services for tests
working-directory: test/java/spark
run: |
echo "=== Starting SeaweedFS Services for Tests ==="
docker compose down -v || true
docker compose build --no-cache
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
echo "Waiting for services..."
for i in {1..30}; do
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
echo "OK SeaweedFS filer is ready!"
break
fi
if [ $i -eq 30 ]; then
echo "FAILED Services failed to start"
docker compose ps -a
docker compose logs
exit 1
fi
echo "Waiting... ($i/30)"
sleep 2
done
curl -f http://localhost:9333/cluster/status || exit 1
echo "OK All services healthy"
- name: Prepare Maven repository for Docker
working-directory: test/java/spark
run: |
echo "Copying Maven artifacts for Docker container..."
mkdir -p .m2/repository/com
cp -r ~/.m2/repository/com/seaweedfs .m2/repository/com/
echo "OK Maven artifacts copied"
- name: Run Spark integration tests
working-directory: test/java/spark
run: |
echo "=== Running Spark Integration Tests ==="
docker compose up --abort-on-container-exit spark-tests
echo ""
echo "=== Test Logs ==="
docker compose logs spark-tests | tail -100
- name: Stop test services
if: always()
working-directory: test/java/spark
run: docker compose down -v
- name: Upload test results
if: always()
uses: actions/upload-artifact@v7
with:
name: spark-test-results
path: test/java/spark/target/surefire-reports/
retention-days: 30
- name: Publish test report
if: always()
uses: dorny/test-reporter@v3
with:
name: Spark Test Results
path: test/java/spark/target/surefire-reports/*.xml
reporter: java-junit
fail-on-error: true
# ========================================
# SPARK EXAMPLE (HOST-BASED) - DISABLED
# Note: Host-based example doesn't work with Docker networking
# because master returns Docker hostnames (seaweedfs-volume)
# which are not resolvable from the host.
# The Docker-based tests above are sufficient.
# ========================================
- name: Cache Apache Spark
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
id: cache-spark
uses: actions/cache@v5
with:
path: spark-3.5.0-bin-hadoop3
key: spark-3.5.0-hadoop3
- name: Download Apache Spark
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && steps.cache-spark.outputs.cache-hit != 'true'
run: |
echo "Downloading Apache Spark 3.5.0..."
wget -q https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz
tar xzf spark-3.5.0-bin-hadoop3.tgz
echo "OK Spark downloaded"
- name: Start SeaweedFS services for example
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: |
echo "=== Starting SeaweedFS Services for Example ==="
docker compose down -v || true
docker compose build --no-cache
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
echo "Waiting for services..."
for i in {1..30}; do
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
echo "OK SeaweedFS filer is ready!"
break
fi
if [ $i -eq 30 ]; then
echo "FAILED Services failed to start"
docker compose ps -a
docker compose logs
exit 1
fi
echo "Waiting... ($i/30)"
sleep 2
done
curl -f http://localhost:9333/cluster/status || exit 1
echo "OK All services healthy"
- name: Clean target directory
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: |
# Force remove target directory to avoid permission issues
sudo rm -rf target || rm -rf target || true
echo "OK Target directory cleaned"
- name: Build project for example
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: mvn package -DskipTests
- name: Run Spark example application
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: |
echo "=== Running Spark Example Application ==="
export SPARK_HOME=$(pwd)/../../../spark-3.5.0-bin-hadoop3
$SPARK_HOME/bin/spark-submit \
--class seaweed.spark.SparkSeaweedFSExample \
--master local[2] \
--conf spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem \
--conf spark.hadoop.fs.seaweed.filer.host=localhost \
--conf spark.hadoop.fs.seaweed.filer.port=8888 \
--conf spark.hadoop.fs.seaweed.filer.port.grpc=18888 \
--conf spark.hadoop.fs.seaweed.replication="" \
target/seaweedfs-spark-integration-tests-1.0-SNAPSHOT.jar \
seaweedfs://localhost:8888/ci-spark-output
echo "OK Example completed"
- name: Verify example output
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
run: |
echo "Verifying output..."
curl -s http://localhost:8888/ci-spark-output/ || echo "Output listing unavailable"
- name: Stop example services
if: false && always() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: docker compose down -v
# ========================================
# DIAGNOSTICS
# ========================================
- name: Display diagnostics on failure
if: failure()
working-directory: test/java/spark
run: |
echo "=== Container Status ==="
docker compose ps -a
echo ""
echo "=== Master Logs ==="
docker compose logs seaweedfs-master
echo ""
echo "=== Volume Logs ==="
docker compose logs seaweedfs-volume
echo ""
echo "=== Filer Logs ==="
docker compose logs seaweedfs-filer
echo ""
echo "=== Volume List ==="
docker compose exec -T seaweedfs-master weed shell <<EOF || echo "Failed"
volume.list
exit
EOF
echo ""
echo "=== Cluster Status ==="
curl -s http://localhost:9333/dir/status | jq '.' || curl -s http://localhost:9333/dir/status
@@ -1,46 +0,0 @@
name: Telemetry Integration Tests
on:
push:
branches: [ master ]
paths:
- 'telemetry/**'
- 'weed/telemetry/**'
- '.github/workflows/telemetry-integration.yml'
pull_request:
branches: [ master ]
paths:
- 'telemetry/**'
- 'weed/telemetry/**'
- '.github/workflows/telemetry-integration.yml'
permissions:
contents: read
jobs:
telemetry-integration-test:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build telemetry server
run: cd telemetry/server && go build -o telemetry-server .
- name: Run telemetry integration test
run: go run telemetry/test/integration.go
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: telemetry-test-logs
path: telemetry-server-test.log
retention-days: 7
@@ -1,111 +0,0 @@
name: "test s3 over https using aws-cli"
on:
push:
branches: [master, test-https-s3-awscli]
pull_request:
branches: [master, test-https-s3-awscli]
env:
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_ENDPOINT_URL: https://localhost:8443
defaults:
run:
working-directory: weed
jobs:
awscli-tests:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
- uses: actions/setup-go@v6
with:
go-version: ^1.25
- name: Build SeaweedFS
run: |
go build
- name: Start SeaweedFS
run: |
set -e
mkdir -p /tmp/data
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
- name: Setup Caddy
run: |
curl -fsSL "https://caddyserver.com/api/download?os=linux&arch=amd64" -o caddy
chmod +x caddy
./caddy version
echo "{
auto_https disable_redirects
local_certs
}
localhost:8443 {
tls internal
reverse_proxy localhost:8333
}" > Caddyfile
- name: Start Caddy
run: |
./caddy start
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
- name: Create Bucket
run: |
aws --no-verify-ssl s3api create-bucket --bucket bucket
- name: Test PutObject
run: |
set -e
dd if=/dev/urandom of=generated bs=1M count=2
aws --no-verify-ssl s3api put-object --bucket bucket --key test-putobject --body generated
aws --no-verify-ssl s3api get-object --bucket bucket --key test-putobject downloaded
diff -q generated downloaded
rm -f generated downloaded
- name: Test Multi-part Upload
run: |
set -e
dd if=/dev/urandom of=generated bs=1M count=32
aws --no-verify-ssl s3 cp --no-progress generated s3://bucket/test-multipart
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded
diff -q generated downloaded
rm -f generated downloaded
- name: Test GetObject with If-Match
run: |
set -e
dd if=/dev/urandom of=generated bs=1M count=32
ETAG=$(aws --no-verify-ssl s3api put-object --bucket bucket --key test-get-obj --body generated | jq -r .ETag)
# jq -r already removes quotes, so use ETAG directly (handles both simple and multipart ETags)
aws --no-verify-ssl s3api get-object --bucket bucket --key test-get-obj --if-match "$ETAG" downloaded
diff -q generated downloaded
rm -f generated downloaded
- name: Show server logs on failure
if: failure()
run: |
echo "========================================="
echo "SeaweedFS Server Logs"
echo "========================================="
# Note: weed.log is relative to working-directory (weed/)
if [ -f weed.log ]; then
cat weed.log
else
echo "No weed.log file found"
fi
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: seaweedfs-logs
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
path: weed/weed.log
retention-days: 3
-56
View File
@@ -1,56 +0,0 @@
name: "TLS Rotation Integration Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
permissions:
contents: read
jobs:
tls-rotation-tests:
name: TLS Rotation Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run TLS Rotation Integration Tests
working-directory: test/tls_rotation
run: |
go test -v -count=1 -timeout 5m
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting master logs from temp directories..."
mkdir -p /tmp/tls-rotation-test-logs
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
fi
done
echo "Collected logs:"
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: tls-rotation-test-logs
path: /tmp/tls-rotation-test-logs/
retention-days: 14
-103
View File
@@ -1,103 +0,0 @@
name: "TUS Protocol Tests"
on:
pull_request:
concurrency:
group: ${{ github.head_ref || github.ref }}/tus-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
tus-integration-tests:
name: TUS Protocol Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run TUS Integration Tests
timeout-minutes: 15
working-directory: test/tus
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting TUS Tests ==="
# Run tests with automatic server management
make test-with-server || {
echo "TUS integration tests failed, checking logs..."
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
echo "=== Filer logs ==="
tail -100 /tmp/seaweedfs-tus-filer.log
fi
if [ -f /tmp/seaweedfs-tus-master.log ]; then
echo "=== Master logs ==="
tail -50 /tmp/seaweedfs-tus-master.log
fi
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
echo "=== Volume logs ==="
tail -50 /tmp/seaweedfs-tus-volume.log
fi
exit 1
}
- name: Show server logs on failure
if: failure()
working-directory: test/tus
run: |
echo "=== Filer Server Logs ==="
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
echo "Last 100 lines of filer logs:"
tail -100 /tmp/seaweedfs-tus-filer.log
else
echo "No filer log file found"
fi
echo "=== Master Server Logs ==="
if [ -f /tmp/seaweedfs-tus-master.log ]; then
tail -50 /tmp/seaweedfs-tus-master.log
else
echo "No master log file found"
fi
echo "=== Volume Server Logs ==="
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
tail -50 /tmp/seaweedfs-tus-volume.log
else
echo "No volume log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp 2>/dev/null | grep -E "(18888|19333|18080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: tus-test-logs
path: |
/tmp/seaweedfs-tus-*.log
retention-days: 3
@@ -1,56 +0,0 @@
name: "Vacuum Integration Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
permissions:
contents: read
jobs:
vacuum-integration-tests:
name: Vacuum Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run Vacuum Integration Tests
working-directory: test/vacuum
run: |
go test -v -timeout 10m
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting server logs from temp directories..."
mkdir -p /tmp/vacuum-test-logs
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
fi
done
echo "Collected logs:"
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: vacuum-integration-test-logs
path: /tmp/vacuum-test-logs/
retention-days: 14
@@ -1,121 +0,0 @@
name: "Volume Server Integration Tests"
on:
pull_request:
branches: [ master ]
paths:
- 'test/volume_server/**'
- 'weed/server/**'
- 'weed/storage/**'
- 'weed/pb/volume_server.proto'
- 'weed/pb/volume_server_pb/**'
- '.github/workflows/volume-server-integration-tests.yml'
push:
branches: [ master, main ]
paths:
- 'test/volume_server/**'
- 'weed/server/**'
- 'weed/storage/**'
- 'weed/pb/volume_server.proto'
- 'weed/pb/volume_server_pb/**'
- '.github/workflows/volume-server-integration-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
cancel-in-progress: true
permissions:
contents: read
env:
TEST_TIMEOUT: '30m'
jobs:
volume-server-integration-tests:
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
runs-on: ubuntu-22.04
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
test-type: [grpc, http]
shard: [1, 2, 3]
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS binary
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Run volume server integration tests
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: |
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
- name: Collect logs on failure
if: failure()
run: |
mkdir -p /tmp/volume-server-it-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: volume-server-integration-test-logs
path: /tmp/volume-server-it-logs/
if-no-files-found: warn
retention-days: 7
- name: Test summary
if: always()
run: |
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
-58
View File
@@ -2,7 +2,6 @@
vendor
tags
*.swp
.claude/
### OSX template
.DS_Store
.AppleDouble
@@ -88,60 +87,3 @@ other/java/hdfs/dependency-reduced-pom.xml
# binary file
weed/weed
docker/weed
# test generated files
weed/*/*.jpg
docker/weed_sub
docker/weed_pub
weed/mq/schema/example.parquet
docker/agent_sub_record
test/mq/bin/consumer
test/mq/bin/producer
test/producer
bin/weed
weed_binary
/test/s3/copying/filerldb2
/filerldb2
/test/s3/retention/test-volume-data
test/s3/cors/weed-test.log
test/s3/cors/weed-server.pid
/test/s3/cors/test-volume-data
test/s3/cors/cors.test
/test/s3/retention/filerldb2
test/s3/retention/weed-server.pid
test/s3/retention/weed-test.log
/test/s3/versioning/test-volume-data
test/s3/versioning/weed-test.log
/docker/admin_integration/data
docker/agent_pub_record
docker/admin_integration/weed-local
/seaweedfs-rdma-sidecar/bin
/test/s3/encryption/filerldb2
/test/s3/sse/filerldb2
test/s3/sse/weed-test.log
ADVANCED_IAM_DEVELOPMENT_PLAN.md
/test/s3/iam/test-volume-data
*.log
weed-iam
test/kafka/kafka-client-loadtest/weed-linux-arm64
/test/tus/filerldb2
coverage.out
/test/s3/remote_cache/test-primary-data
/test/s3/remote_cache/test-remote-data
test/s3/remote_cache/remote-server.pid
test/s3/remote_cache/primary-server.pid
/test/erasure_coding/filerldb2
/test/s3/cors/test-mini-data
/test/s3/filer_group/test-volume-data
# ID and PID files
*.id
*.pid
test/s3/iam/.test_env
/test/erasure_coding/admin_dockertest/tmp
/test/erasure_coding/admin_dockertest/task_logs
weed_bin
telemetry/server/telemetry-server
.aider*
/seaweed-volume/docs
-5
View File
@@ -1,5 +0,0 @@
{
"setup": [],
"teardown": [],
"run": []
}
-74
View File
@@ -1,74 +0,0 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
- Using welcoming and inclusive language
- Being respectful of differing viewpoints and experiences
- Gracefully accepting constructive criticism
- Focusing on what is best for the community
- Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
- The use of sexualized language or imagery and unwelcome sexual attention or
advances
- Trolling, insulting/derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or electronic
address, without explicit permission
- Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at <enteremailhere>. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at [http://contributor-covenant.org/version/1/4][version]
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
+1 -1
View File
@@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2025 Chris Lu
Copyright 2016 Chris Lu
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
+5 -64
View File
@@ -1,73 +1,14 @@
.PHONY: test admin-generate admin-build admin-clean admin-dev admin-run admin-test admin-fmt admin-help weed-commands
BINARY = weed
ADMIN_DIR = weed/admin
SOURCE_DIR = .
debug ?= 0
all: install
install: admin-generate
install:
cd weed; go install
weed-commands:
cd weed && $(MAKE) weed-db weed-sql
full_install:
cd weed; go install -tags "elastic gocdk sqlite ydb tikv"
warp_install:
go install github.com/minio/warp@v0.7.6
full_install: admin-generate
cd weed; go install -tags "elastic gocdk sqlite ydb tarantool tikv rclone"
server: install
weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
benchmark: install warp_install
pkill weed || true
pkill warp || true
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
warp client &
while ! nc -z localhost 8000 ; do sleep 1 ; done
warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
pkill warp
pkill weed
# curl -o profile "http://127.0.0.1:6060/debug/pprof/profile?debug=1"
benchmark_with_pprof: debug = 1
benchmark_with_pprof: benchmark
test: admin-generate
cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
# Admin component targets
admin-generate:
@cd $(ADMIN_DIR) && $(MAKE) generate
admin-build: admin-generate
@echo "Building admin component..."
@cd $(ADMIN_DIR) && $(MAKE) build
admin-clean:
@echo "Cleaning admin component..."
@cd $(ADMIN_DIR) && $(MAKE) clean
admin-dev:
@echo "Starting admin development server..."
@cd $(ADMIN_DIR) && $(MAKE) dev
admin-run:
@echo "Running admin server..."
@cd $(ADMIN_DIR) && $(MAKE) run
admin-test:
@echo "Testing admin component..."
@cd $(ADMIN_DIR) && $(MAKE) test
admin-fmt:
@echo "Formatting admin component..."
@cd $(ADMIN_DIR) && $(MAKE) fmt
admin-help:
@echo "Admin component help..."
@cd $(ADMIN_DIR) && $(MAKE) help
tests:
cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
+61 -133
View File
@@ -3,12 +3,12 @@
[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
[![Build Status](https://img.shields.io/github/actions/workflow/status/seaweedfs/seaweedfs/go.yml)](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
[![Build Status](https://img.shields.io/github/workflow/status/chrislusf/seaweedfs/Go)](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
[![GoDoc](https://godoc.org/github.com/seaweedfs/seaweedfs/weed?status.svg)](https://godoc.org/github.com/seaweedfs/seaweedfs/weed)
[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/seaweedfs/seaweedfs/wiki)
[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/seaweedfs)](https://artifacthub.io/packages/search?repo=seaweedfs)
![SeaweedFS Logo](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/seaweedfs.png)
@@ -32,9 +32,8 @@ Your support will be really appreciated by me and other supporters!
-->
### Gold Sponsors
[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
[![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
[![keepsec](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/keepsec.png)](https://www.keepsec.ca)
---
@@ -46,7 +45,6 @@ Your support will be really appreciated by me and other supporters!
- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
- [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki)
- [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
- [SeaweedFS Introduction Slides 2025.5](https://docs.google.com/presentation/d/1tdkp45J01oRV68dIm4yoTXKJDof-EhainlA0LMXexQE/edit?usp=sharing)
- [SeaweedFS Introduction Slides 2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing)
- [SeaweedFS Introduction Slides 2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
@@ -54,8 +52,9 @@ Table of Contents
=================
* [Quick Start](#quick-start)
* [Quick Start with weed mini](#quick-start-with-weed-mini)
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
* [Quick Start with Single Binary](#quick-start-with-single-binary)
* [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws)
* [Introduction](#introduction)
* [Features](#features)
* [Additional Features](#additional-features)
@@ -67,53 +66,26 @@ Table of Contents
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
* [Compared to GlusterFS](#compared-to-glusterfs)
* [Compared to Ceph](#compared-to-ceph)
* [Compared to Minio](#compared-to-minio)
* [Dev Plan](#dev-plan)
* [Installation Guide](#installation-guide)
* [Disk Related Topics](#disk-related-topics)
* [Benchmark](#benchmark)
* [Enterprise](#enterprise)
* [License](#license)
# Quick Start #
## Quick Start with weed mini ##
Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file, or run `go install github.com/seaweedfs/seaweedfs/weed@latest`. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
```bash
AWS_ACCESS_KEY_ID=admin \
AWS_SECRET_ACCESS_KEY=secret \
S3_BUCKET=my-bucket \
./weed mini -dir=/data
```
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
The same command starts everything else too:
- **S3 Endpoint**: http://localhost:8333
- **Master UI**: http://localhost:9333
- **Volume Server**: http://localhost:9340
- **Filer UI**: http://localhost:8888
- **WebDAV**: http://localhost:7333
- **Admin UI**: http://localhost:23646
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
Perfect for development, testing, learning SeaweedFS, and single-node deployments. To scale out, add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, on another machine, or on thousands of machines.
## Quick Start for S3 API on Docker ##
```bash
docker run -p 8333:8333 \
-e AWS_ACCESS_KEY_ID=admin \
-e AWS_SECRET_ACCESS_KEY=secret \
-e S3_BUCKET=my-bucket \
chrislusf/seaweedfs
```
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
Same behavior as the `weed mini` command above — the S3 endpoint is at http://localhost:8333 with `my-bucket` pre-created. Drop the env vars to run anonymously for development.
## Quick Start with Single Binary ##
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
## Quick Start SeaweedFS S3 on AWS ##
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
# Introduction #
@@ -122,7 +94,7 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
1. to store billions of files!
2. to serve the files fast!
SeaweedFS started as a blob store to handle small files efficiently.
SeaweedFS started as an Object Store to handle small files efficiently.
Instead of managing all file metadata in a central master,
the central master only manages volumes on volume servers,
and these volume servers manage files and their metadata.
@@ -134,33 +106,32 @@ It is so simple with O(1) disk reads that you are welcome to challenge the perfo
SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
Also, SeaweedFS implements erasure coding with ideas from
[f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and has a lot of similarities with [Facebook’s Tectonic Filesystem](https://www.usenix.org/system/files/fast21-pan.pdf) and [Google's Colossus File System](https://cloud.google.com/blog/products/storage-data-transfer/a-peek-behind-colossus-googles-file-system)
[f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and has a lot of similarities with [Facebook’s Tectonic Filesystem](https://www.usenix.org/system/files/fast21-pan.pdf)
On top of the blob store, optional [Filer] can support directories and POSIX attributes.
On top of the object store, optional [Filer] can support directories and POSIX attributes.
Filer is a separate linearly-scalable stateless server with customizable metadata stores,
e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, YDB, etc.
For any distributed key value stores, the large values can be offloaded to SeaweedFS.
With the fast access speed and linearly scalable capacity,
SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore].
SeaweedFS can transparently integrate with the cloud.
With hot data on local cluster, and warm data on the cloud with O(1) access time,
SeaweedFS can achieve both fast local access time and elastic cloud storage capacity.
What's more, the cloud storage access API cost is minimized.
Faster and cheaper than direct cloud storage!
SeaweedFS also ships a built-in **Iceberg REST Catalog**, turning the same cluster into a self-contained lakehouse.
Spark, Trino, Dremio, DuckDB, and RisingWave can query Iceberg tables directly — no Hive Metastore, Glue, or
external catalog service required. Storage and table metadata live in one system, simplifying on-prem and
small-team analytics stacks.
Faster and Cheaper than direct cloud storage!
[Back to TOC](#table-of-contents)
# Features #
## Additional Blob Store Features ##
* Support different replication levels, with rack and data center aware.
## Additional Features ##
* Can choose no replication or different replication levels, rack and data center aware.
* Automatic master servers failover - no single point of failure (SPOF).
* Automatic compression depending on file MIME type.
* Automatic Gzip compression depending on file MIME type.
* Automatic compaction to reclaim disk space after deletion or update.
* [Automatic entry TTL expiration][VolumeServerTTL].
* Flexible Capacity Expansion: Any server with some disk space can add to the total storage space.
* Any server with some disk spaces can add to the total storage space.
* Adding/Removing servers does **not** cause any data re-balancing unless triggered by admin commands.
* Optional picture resizing.
* Support ETag, Accept-Range, Last-Modified, etc.
@@ -168,12 +139,12 @@ small-team analytics stacks.
* Support rebalancing the writable and readonly volumes.
* [Customizable Multiple Storage Tiers][TieredStorage]: Customizable storage disk types to balance performance and cost.
* [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data.
* [Erasure Coding for warm storage][ErasureCoding] Rack-Aware 10.4 erasure coding reduces storage cost and increases availability. Enterprise version can customize EC ratio.
* [Erasure Coding for warm storage][ErasureCoding] Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
[Back to TOC](#table-of-contents)
## Filer Features ##
* [Filer server][Filer] provides "normal" directories and files via HTTP.
* [Filer server][Filer] provides "normal" directories and files via http.
* [File TTL][FilerTTL] automatically expires file metadata and actual file data.
* [Mount filer][Mount] reads and writes files directly as a local directory via FUSE.
* [Filer Store Replication][FilerStoreReplication] enables HA for filer meta data stores.
@@ -187,13 +158,6 @@ small-team analytics stacks.
* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
## Data Lakehouse Features ##
* [S3 Table Buckets][S3TableBucket] expose a dedicated namespace for Iceberg tables with strict layout validation.
* Built-in [Iceberg REST Catalog][IcebergCatalog] runs alongside the S3 endpoint — no external metastore needed.
* Native integrations with [Apache Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], and [RisingWave][RisingWaveIceberg].
* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan removal, manifest rewriting.
* Granular IAM at the bucket, namespace, and table level via standard S3 bucket policies.
## Kubernetes ##
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
@@ -217,19 +181,11 @@ small-team analytics stacks.
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
[S3TableBucket]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Table-Bucket
[IcebergCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Iceberg-Catalog
[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
[SparkIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Spark-Iceberg-Integration
[TrinoIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Trino-Iceberg-Integration
[DremioIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Dremio-Iceberg-Integration
[DuckDBIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/DuckDB-Iceberg-Integration
[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
[Back to TOC](#table-of-contents)
## Example: Using Seaweed Blob Store ##
## Example: Using Seaweed Object Store ##
By default, the master node runs on port 9333, and the volume nodes run on port 8080.
Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
@@ -245,29 +201,27 @@ SeaweedFS uses HTTP REST operations to read, write, and delete. The responses ar
### Start Volume Servers ###
```
> weed volume -dir="/tmp/data1" -max=5 -master="localhost:9333" -port=8080 &
> weed volume -dir="/tmp/data2" -max=10 -master="localhost:9333" -port=8081 &
> weed volume -dir="/tmp/data1" -max=5 -mserver="localhost:9333" -port=8080 &
> weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
```
### Write A Blob ###
### Write File ###
A blob, also referred as a needle, a chunk, or mistakenly as a file, is just a byte array. It can have attributes, such as name, mime type, create or update time, etc. But basically it is just a byte array of a relatively small size, such as 2 MB ~ 64 MB. The size is not fixed.
To upload a blob: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server URL:
To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server URL:
```
> curl http://localhost:9333/dir/assign
{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
```
Second, to store the blob content, send a HTTP multi-part POST request to `url + '/' + fid` from the response:
Second, to store the file content, send a HTTP multi-part POST request to `url + '/' + fid` from the response:
```
> curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
{"name":"myphoto.jpg","size":43234,"eTag":"1cc0118e"}
```
To update, send another POST request with updated blob content.
To update, send another POST request with updated file content.
For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
@@ -275,7 +229,7 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
> curl -X DELETE http://127.0.0.1:8080/3,01637037d6
```
### Save Blob Id ###
### Save File Id ###
Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
@@ -287,9 +241,9 @@ The file key and file cookie are both coded in hex. You can store the <volume id
If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most uses will not need 2^32 volumes.
If space is really a concern, you can store the file id in the binary format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
### Read a Blob ###
### Read File ###
Here is an example of how to render the URL.
@@ -330,7 +284,7 @@ http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fill
### Rack-Aware and Data Center-Aware Replication ###
SeaweedFS applies the replication strategy at a volume level. So, when you are getting a blob id, you can specify the replication strategy. For example:
SeaweedFS applies the replication strategy at a volume level. So, when you are getting a file id, you can specify the replication strategy. For example:
```
curl http://localhost:9333/dir/assign?replication=001
@@ -353,7 +307,7 @@ More details about replication can be found [on the wiki][Replication].
You can also set the default replication strategy when starting the master server.
### Allocate Blob Key on Specific Data Center ###
### Allocate File Key on Specific Data Center ###
Volume servers can be started with a specific data center name:
@@ -362,7 +316,7 @@ Volume servers can be started with a specific data center name:
weed volume -dir=/tmp/2 -port=8081 -dataCenter=dc2
```
When requesting a blob key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specifies that the assigned volume should be limited to 'dc1':
When requesting a file key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specifies that the assigned volume should be limited to 'dc1':
```
http://localhost:9333/dir/assign?dataCenter=dc1
@@ -381,15 +335,15 @@ When requesting a blob key, an optional "dataCenter" parameter can limit the ass
[Back to TOC](#table-of-contents)
## Blob Store Architecture ##
## Object Store Architecture ##
Usually distributed file systems split each file into chunks. A central server keeps a mapping of filenames to chunks, and also which chunks each chunk server has.
Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
The main drawback is that the central server can't handle many small files efficiently, and since all read requests need to go through the central master, so it might not scale well for many concurrent users.
The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, so it might not scale well for many concurrent users.
Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of blobs. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
The actual blob metadata, which are the blob volume, offset, and size, is stored in each volume on volume servers. Since each volume server only manages metadata of blobs on its own disk, with only 16 bytes for each blob, all access can read the metadata just from memory and only needs one disk operation to actually read file data.
The actual file metadata is stored in each volume on volume servers. Since each volume server only manages metadata of files on its own disk, with only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
For comparison, consider that an xfs inode structure in Linux is 536 bytes.
@@ -403,13 +357,23 @@ On each write request, the master server also generates a file key, which is a g
### Write and Read files ###
When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node URL) for the blob. The client then contacts the volume node and POSTs the blob content.
When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node URL) for the file. The client then contacts the volume node and POSTs the file content.
When a client needs to read a blob based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node URL, volume node public URL), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node URL, volume node public URL), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
Please see the example for details on the write-read process.
### Storage Size ###
In the current implementation, each volume can hold 32 gibibytes (32GiB or 8x2^32 bytes). This is because we align content to 8 bytes. We can easily increase this to 64GiB, or 128GiB, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
There can be 4 gibibytes (4GiB or 2^32 bytes) of volumes. So the total system size is 8 x 4GiB x 4GiB which is 128 exbibytes (128EiB or 2^67 bytes).
Each individual file size is limited to the volume size.
### Saving memory ###
All blob metadata stored on a volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
### Tiered Storage to the cloud ###
@@ -423,12 +387,6 @@ If the hot/warm data is split as 20/80, with 20 servers, you can achieve storage
[Back to TOC](#table-of-contents)
## SeaweedFS Filer ##
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory sturcture is an interface that is implemented in many key-value stores or databases.
The content of a file is mapped to one or many blobs, distributed to multiple volumes on multiple volume servers.
## Compared to Other File Systems ##
Most other distributed file systems seem more complicated than necessary.
@@ -557,8 +515,6 @@ cd seaweedfs/weed && make install
Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
For more installation options, including how to run with Docker, see the [Getting Started guide](https://github.com/seaweedfs/seaweedfs/wiki/Getting-Started).
[Back to TOC](#table-of-contents)
## Disk Related Topics ##
@@ -625,35 +581,6 @@ Percentage of the requests served within a certain time (ms)
100% 54.1 ms
```
### Run WARP and launch a mixed benchmark. ###
```
make benchmark
warp: Benchmark data written to "warp-mixed-2025-12-05[194844]-kBpU.csv.zst"
Mixed operations.
Operation: DELETE, 10%, Concurrency: 20, Ran 42s.
* Throughput: 55.13 obj/s
Operation: GET, 45%, Concurrency: 20, Ran 42s.
* Throughput: 2477.45 MiB/s, 247.75 obj/s
Operation: PUT, 15%, Concurrency: 20, Ran 42s.
* Throughput: 825.85 MiB/s, 82.59 obj/s
Operation: STAT, 30%, Concurrency: 20, Ran 42s.
* Throughput: 165.27 obj/s
Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
```
[Back to TOC](#table-of-contents)
## Enterprise ##
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
which has a self-healing storage format with better data protection.
[Back to TOC](#table-of-contents)
## License ##
@@ -675,4 +602,5 @@ The text of this page is available for modification and reuse under the terms of
[Back to TOC](#table-of-contents)
## Stargazers over time
[![Stargazers over time](https://starchart.cc/seaweedfs/seaweedfs.svg?variant=adaptive)](https://starchart.cc/seaweedfs/seaweedfs)
[![Stargazers over time](https://starchart.cc/chrislusf/seaweedfs.svg)](https://starchart.cc/chrislusf/seaweedfs)
File diff suppressed because it is too large Load Diff
-24
View File
@@ -1,24 +0,0 @@
# Security Policy
## Reporting a Vulnerability
If you find a security issue in SeaweedFS, please report it privately:
- Email: support@seaweedfs.com
- Do not open a public GitHub issue
Please include:
- A clear description of the issue
- Steps to reproduce (if possible)
- Affected versions
## Response
- We will respond as soon as possible (usually within 1 business day)
- We will investigate and work on a fix
- We may coordinate disclosure with you
## Notes
- Please allow time for a fix before public disclosure
- If you’re unsure whether something is a security issue, feel free to reach out
-790
View File
@@ -1,790 +0,0 @@
# Execution Plan: SeaweedFS Volume Server — Go to Rust Port
## Scope Summary
| Component | Go Source | Lines (non-test) | Description |
|---|---|---|---|
| CLI & startup | `weed/command/volume.go` | 476 | ~40 CLI flags, server bootstrap |
| HTTP server + handlers | `weed/server/volume_server*.go` | 1,517 | Struct, routes, read/write/delete handlers |
| gRPC handlers | `weed/server/volume_grpc_*.go` | 3,073 | 40 RPC method implementations |
| Storage engine | `weed/storage/` | 15,271 | Volumes, needles, index, compaction, EC, backend |
| Protobuf definitions | `weed/pb/volume_server.proto` | 759 | Service + message definitions |
| Shared utilities | `weed/security/`, `weed/stats/`, `weed/util/` | ~2,000+ | JWT, TLS, metrics, helpers |
| **Total** | | **~23,000+** | |
## Rust Crate & Dependency Strategy
```
seaweed-volume/
├── Cargo.toml
├── build.rs # protobuf codegen
├── proto/
│ ├── volume_server.proto # copied from Go, adapted
│ └── remote.proto
├── src/
│ ├── main.rs # CLI entry point
│ ├── config.rs # CLI flags + config
│ ├── server/
│ │ ├── mod.rs
│ │ ├── volume_server.rs # VolumeServer struct + lifecycle
│ │ ├── http_handlers.rs # HTTP route dispatch
│ │ ├── http_read.rs # GET/HEAD handlers
│ │ ├── http_write.rs # POST/PUT handlers
│ │ ├── http_delete.rs # DELETE handler
│ │ ├── http_admin.rs # /status, /healthz, /ui
│ │ ├── grpc_service.rs # gRPC trait impl dispatch
│ │ ├── grpc_vacuum.rs
│ │ ├── grpc_copy.rs
│ │ ├── grpc_erasure_coding.rs
│ │ ├── grpc_tail.rs
│ │ ├── grpc_admin.rs
│ │ ├── grpc_read_write.rs
│ │ ├── grpc_batch_delete.rs
│ │ ├── grpc_scrub.rs
│ │ ├── grpc_tier.rs
│ │ ├── grpc_remote.rs
│ │ ├── grpc_query.rs
│ │ ├── grpc_state.rs
│ │ └── grpc_client_to_master.rs # heartbeat
│ ├── storage/
│ │ ├── mod.rs
│ │ ├── store.rs # Store (multi-disk manager)
│ │ ├── volume.rs # Volume struct + lifecycle
│ │ ├── volume_read.rs
│ │ ├── volume_write.rs
│ │ ├── volume_compact.rs
│ │ ├── volume_info.rs
│ │ ├── needle/
│ │ │ ├── mod.rs
│ │ │ ├── needle.rs # Needle struct + serialization
│ │ │ ├── needle_read.rs
│ │ │ ├── needle_write.rs
│ │ │ ├── needle_map.rs # in-memory NeedleMap
│ │ │ ├── needle_value.rs
│ │ │ └── crc.rs
│ │ ├── super_block.rs
│ │ ├── idx/
│ │ │ ├── mod.rs
│ │ │ └── idx.rs # .idx file format read/write
│ │ ├── needle_map_leveldb.rs
│ │ ├── types.rs # NeedleId, Offset, Size, DiskType
│ │ ├── disk_location.rs # DiskLocation per-directory
│ │ ├── erasure_coding/
│ │ │ ├── mod.rs
│ │ │ ├── ec_volume.rs
│ │ │ ├── ec_shard.rs
│ │ │ ├── ec_encoder.rs # Reed-Solomon encoding
│ │ │ └── ec_decoder.rs
│ │ └── backend/
│ │ ├── mod.rs
│ │ ├── disk.rs
│ │ └── s3_backend.rs # tiered storage to S3
│ ├── topology/
│ │ └── volume_layout.rs # replication placement
│ ├── security/
│ │ ├── mod.rs
│ │ ├── guard.rs # whitelist + JWT gate
│ │ ├── jwt.rs
│ │ └── tls.rs
│ ├── stats/
│ │ ├── mod.rs
│ │ └── metrics.rs # Prometheus counters/gauges
│ └── util/
│ ├── mod.rs
│ ├── grpc.rs
│ ├── http.rs
│ └── file.rs
└── tests/
├── integration/
│ ├── http_read_test.rs
│ ├── http_write_test.rs
│ ├── grpc_test.rs
│ └── storage_test.rs
└── unit/
├── needle_test.rs
├── idx_test.rs
├── super_block_test.rs
└── ec_test.rs
```
### Key Rust dependencies
| Purpose | Crate |
|---|---|
| Async runtime | `tokio` |
| gRPC | `tonic` + `prost` |
| HTTP server | `hyper` + `axum` |
| CLI parsing | `clap` (derive) |
| Prometheus metrics | `prometheus` |
| JWT | `jsonwebtoken` |
| TLS | `rustls` + `tokio-rustls` |
| LevelDB | `rusty-leveldb` or `rocksdb` |
| Reed-Solomon EC | `reed-solomon-erasure` |
| Logging | `tracing` + `tracing-subscriber` |
| Config (security.toml) | `toml` + `serde` |
| CRC32 | `crc32fast` |
| Memory-mapped files | `memmap2` |
---
## Phased Execution Plan
### Phase 1: Project Skeleton & Protobuf Codegen
**Goal:** Cargo project compiles, proto codegen works, CLI parses all flags.
**Steps:**
1.1. Create `seaweed-volume/Cargo.toml` with all dependencies listed above.
1.2. Copy `volume_server.proto` and `remote.proto` into `proto/`. Adjust package paths for Rust codegen.
1.3. Create `build.rs` using `tonic-build` to compile `.proto` files into Rust types.
1.4. Create `src/main.rs` with `clap` derive structs mirroring all 40 CLI flags from `weed/command/volume.go`:
- `--port` (default 8080)
- `--port.grpc` (default 0 → 10000+port)
- `--port.public` (default 0 → same as port)
- `--ip` (auto-detect)
- `--id` (default empty → ip:port)
- `--publicUrl`
- `--ip.bind`
- `--master` (default "localhost:9333")
- `--mserver` (deprecated compat)
- `--preStopSeconds` (default 10)
- `--idleTimeout` (default 30)
- `--dataCenter`
- `--rack`
- `--index` [memory|leveldb|leveldbMedium|leveldbLarge]
- `--disk` [hdd|ssd|<tag>]
- `--tags`
- `--dir` (default temp dir)
- `--dir.idx`
- `--max` (default "8")
- `--whiteList`
- `--minFreeSpacePercent` (default "1")
- `--minFreeSpace`
- `--images.fix.orientation` (default false)
- `--readMode` [local|proxy|redirect] (default "proxy")
- `--cpuprofile`
- `--memprofile`
- `--compactionMBps` (default 0)
- `--maintenanceMBps` (default 0)
- `--fileSizeLimitMB` (default 256)
- `--concurrentUploadLimitMB` (default 0)
- `--concurrentDownloadLimitMB` (default 0)
- `--pprof` (default false)
- `--metricsPort` (default 0)
- `--metricsIp`
- `--inflightUploadDataTimeout` (default 60s)
- `--inflightDownloadDataTimeout` (default 60s)
- `--hasSlowRead` (default true)
- `--readBufferSizeMB` (default 4)
- `--index.leveldbTimeout` (default 0)
- `--debug` (default false)
- `--debug.port` (default 6060)
1.5. Implement the same flag validation logic from `startVolumeServer()`:
- Parse comma-separated `--dir`, `--max`, `--minFreeSpace`, `--disk`, `--tags`
- Replicate single-value-to-all-dirs expansion
- Validate count matches between dirs and limits
- `--mserver` backward compat
1.6. **Test:** `cargo build` succeeds. `cargo run -- --help` shows all flags. Proto types generated.
**Verification:** Run with `--port 8080 --dir /tmp --master localhost:9333` — should parse without error and print config.
---
### Phase 2: Core Storage Types & On-Disk Format
**Goal:** Read and write the SeaweedFS needle/volume binary format bit-for-bit compatible with Go.
**Source files to port:**
- `weed/storage/types/needle_types.go` → `src/storage/types.rs`
- `weed/storage/needle/needle.go` → `src/storage/needle/needle.rs`
- `weed/storage/needle/needle_read.go` → `src/storage/needle/needle_read.rs`
- `weed/storage/needle/needle_write.go` (partial) → `src/storage/needle/needle_write.rs`
- `weed/storage/needle/crc.go` → `src/storage/needle/crc.rs`
- `weed/storage/needle/needle_value_map.go` → `src/storage/needle/needle_value.rs`
- `weed/storage/super_block/super_block.go` → `src/storage/super_block.rs`
- `weed/storage/idx/` → `src/storage/idx/`
**Steps:**
2.1. **Fundamental types** (`types.rs`):
- `NeedleId` (u64), `Offset` (u32 or u64 depending on version), `Size` (i32, negative = deleted)
- `Cookie` (u32)
- `DiskType` enum (HDD, SSD, Custom)
- Version constants (Version1=1, Version2=2, Version3=3, CurrentVersion=3)
- Byte serialization matching Go's `binary.BigEndian` encoding
2.2. **SuperBlock** (`super_block.rs`):
- 8-byte header: Version(1) + ReplicaPlacement(1) + TTL(2) + CompactRevision(2) + Reserved(2)
- `ReplicaPlacement` struct with same/diff rack/dc counts
- `TTL` struct with count + unit
- Read/write from first 8 bytes of `.dat` file
- Match exact byte layout from `super_block.go`
2.3. **Needle binary format** (`needle.rs`, `needle_read.rs`):
- Version 2/3 header: Cookie(4) + NeedleId(8) + Size(4)
- Body: Data, Flags, Name, Mime, PairsSize, Pairs, LastModified, TTL, Checksum, AppendAtNs, Padding
- CRC32 checksum (matching Go's `crc32.ChecksumIEEE`)
- Padding to 8-byte alignment
- Read path: read header → compute body length → read body → verify CRC
2.4. **Idx file format** (`idx/`):
- Fixed 16-byte records: NeedleId(8) + Offset(4) + Size(4)
- Sequential append-only file
- Walk/iterate all entries
- Binary search not used (loaded into memory map)
2.5. **NeedleMap (in-memory)** (`needle_map.rs`):
- HashMap<NeedleId, NeedleValue> where NeedleValue = {Offset, Size}
- Load from `.idx` file on volume mount
- Support Get, Set, Delete operations
- Track file count, deleted count, deleted byte count
2.6. **Tests:**
- Unit test: write a needle to bytes → read it back → verify fields match
- Unit test: write/read SuperBlock round-trip
- Unit test: write/read idx entries round-trip
- **Cross-compat test:** Use Go volume server to create a small volume with known data. Read it from Rust and verify all needles decoded correctly. (Keep test fixture `.dat`/`.idx` files in `tests/fixtures/`)
---
### Phase 3: Volume Struct & Lifecycle
**Goal:** Mount, read from, write to, and unmount a volume.
**Source files to port:**
- `weed/storage/volume.go` → `src/storage/volume.rs`
- `weed/storage/volume_read.go` → `src/storage/volume_read.rs`
- `weed/storage/volume_write.go` → `src/storage/volume_write.rs`
- `weed/storage/volume_loading.go`
- `weed/storage/volume_vacuum.go` → `src/storage/volume_compact.rs`
- `weed/storage/volume_info/volume_info.go` → `src/storage/volume_info.rs`
- `weed/storage/volume_super_block.go`
**Steps:**
3.1. **Volume struct** (`volume.rs`):
- Fields: Id, dir, dataFile, nm (NeedleMap), SuperBlock, readOnly, lastModifiedTs, lastCompactIndexOffset, lastCompactRevision
- `noWriteOrDelete` / `noWriteCanDelete` / `readOnly` state flags
- File handles for `.dat` file (read + append)
- Lock strategy: `RwLock` for concurrent reads, exclusive writes
3.2. **Volume loading** — exact logic from `volume_loading.go`:
- Open `.dat` file, read SuperBlock from first 8 bytes
- Load `.idx` file into NeedleMap
- Handle `.vif` (VolumeInfo) JSON sidecar file
- Set volume state based on SuperBlock + VolumeInfo
3.3. **Volume read** (`volume_read.rs`) — from `volume_read.go`:
- `ReadNeedle(needleId, cookie)`: lookup in NeedleMap → seek in .dat → read needle bytes → verify cookie + CRC → return data
- Handle deleted needles (Size < 0)
- `ReadNeedleBlob(offset, size)`: raw blob read
- `ReadNeedleMeta(needleId, offset, size)`: read metadata only
3.4. **Volume write** (`volume_write.rs`) — from `volume_write.go`:
- `WriteNeedle(needle)`: serialize needle → append to .dat → update .idx → update NeedleMap
- `DeleteNeedle(needleId)`: mark as deleted in NeedleMap + append tombstone to .idx
- File size limit check
- Concurrent write serialization (mutex on write path)
3.5. **Volume compaction** (`volume_compact.rs`) — from `volume_vacuum.go`:
- `CheckCompact()`: compute garbage ratio
- `Compact()`: create new .dat/.idx, copy only live needles, update compact revision
- `CommitCompact()`: rename compacted files over originals
- `CleanupCompact()`: remove temp files
- Throttle by `compactionBytePerSecond`
3.6. **Volume info** (`volume_info.rs`):
- Read/write `.vif` JSON sidecar
- VolumeInfo protobuf struct mapping
- Remote file references for tiered storage
3.7. **Tests:**
- Mount a volume, write 100 needles, read them all back, verify content
- Delete 50 needles, verify they return "deleted"
- Compact, verify only 50 remain, verify content
- Read Go-created volume fixtures
---
### Phase 4: Store (Multi-Volume, Multi-Disk Manager)
**Goal:** Manage multiple volumes across multiple disk directories.
**Source files to port:**
- `weed/storage/store.go` → `src/storage/store.rs`
- `weed/storage/disk_location.go` → `src/storage/disk_location.rs`
- `weed/storage/store_ec.go`
- `weed/storage/store_state.go`
**Steps:**
4.1. **DiskLocation** (`disk_location.rs`):
- Directory path, max volume count, min free space, disk type, tags
- Load all volumes from directory on startup
- Track free space, check writable
4.2. **Store** (`store.rs`):
- Vector of `DiskLocation`s
- `GetVolume(volumeId)` → lookup across all locations
- `HasVolume(volumeId)` check
- `AllocateVolume(...)` — create new volume in appropriate location
- `DeleteVolume(...)`, `MountVolume(...)`, `UnmountVolume(...)`
- `DeleteCollection(collection)` — delete all volumes of a collection
- Collect volume status for heartbeat
- `SetStopping()`, `Close()`
- Persistent state (maintenance mode) via `store_state.go`
4.3. **Store state** — `VolumeServerState` protobuf with maintenance flag, persisted to disk.
4.4. **Tests:**
- Create store with 2 dirs, allocate volumes in each, verify load balancing
- Mount/unmount/delete lifecycle
- State persistence across restart
---
### Phase 5: Erasure Coding
**Goal:** Full EC shard encode/decode/read/write/rebuild.
**Source files to port:**
- `weed/storage/erasure_coding/` (3,599 lines)
**Steps:**
5.1. **EC volume + shard structs** — `EcVolume`, `EcShard` with file handles for `.ec00`–`.ec13` shard files + `.ecx` index + `.ecj` journal.
5.2. **EC encoder** — Reed-Solomon 10+4 (configurable) encoding using `reed-solomon-erasure` crate:
- `VolumeEcShardsGenerate`: read .dat → split into data shards → compute parity → write .ec00-.ec13 + .ecx
5.3. **EC decoder/reader** — reconstruct data from any 10 of 14 shards:
- `EcShardRead`: read range from a specific shard
- Locate needle in EC volume via .ecx index
- Handle cross-shard needle reads
5.4. **EC shard operations:**
- Copy, delete, mount, unmount shards
- `VolumeEcShardsRebuild`: rebuild missing shards from remaining
- `VolumeEcShardsToVolume`: reconstruct .dat from EC shards
- `VolumeEcBlobDelete`: mark deleted in EC journal
- `VolumeEcShardsInfo`: report shard metadata
5.5. **Tests:**
- Encode a volume → verify 14 shards created
- Delete 4 shards → rebuild → verify data intact
- Read individual needles from EC volume
- Cross-compat with Go-generated EC shards
---
### Phase 6: Backend / Tiered Storage
**Goal:** Support tiered storage to remote backends (S3, etc).
**Source files to port:**
- `weed/storage/backend/` (1,850 lines)
**Steps:**
6.1. **Backend trait** — abstract `BackendStorage` trait with `ReadAt`, `WriteAt`, `Truncate`, `Close`, `Name`.
6.2. **Disk backend** — default local disk implementation.
6.3. **S3 backend** — upload .dat to S3, read ranges via S3 range requests.
6.4. **Tier move operations:**
- `VolumeTierMoveDatToRemote`: upload .dat to remote, optionally delete local
- `VolumeTierMoveDatFromRemote`: download .dat from remote
6.5. **Tests:**
- Disk backend read/write round-trip
- S3 backend with mock/localstack
---
### Phase 7: Security Layer
**Goal:** JWT authentication, whitelist guard, TLS configuration.
**Source files to port:**
- `weed/security/guard.go` → `src/security/guard.rs`
- `weed/security/jwt.go` → `src/security/jwt.rs`
- `weed/security/tls.go` → `src/security/tls.rs`
**Steps:**
7.1. **Guard** (`guard.rs`):
- Whitelist IP check (exact match on `r.RemoteAddr`)
- Wrap handlers with whitelist enforcement
- `UpdateWhiteList()` for live reload
7.2. **JWT** (`jwt.rs`):
- `SeaweedFileIdClaims` with `fid` field
- Sign with HMAC-SHA256
- Verify + decode with expiry check
- Separate signing keys for read vs write
- `GetJwt(request)` — extract from `Authorization: Bearer` header or `jwt` query param
7.3. **TLS** (`tls.rs`):
- Load server TLS cert/key for gRPC and HTTPS
- Load client TLS for mutual TLS
- Read from `security.toml` config (same format as Go's viper config)
7.4. **Tests:**
- JWT sign → verify round-trip
- JWT with wrong key → reject
- JWT with expired token → reject
- JWT fid mismatch → reject
- Whitelist allow/deny
---
### Phase 8: Prometheus Metrics
**Goal:** Export same metric names as Go for dashboard compatibility.
**Source files to port:**
- `weed/stats/metrics.go` (volume server counters/gauges/histograms)
**Steps:**
8.1. Define all Prometheus metrics matching Go names:
- `VolumeServerRequestCounter` (labels: method, status)
- `VolumeServerRequestHistogram` (labels: method)
- `VolumeServerInFlightRequestsGauge` (labels: method)
- `VolumeServerInFlightUploadSize`
- `VolumeServerInFlightDownloadSize`
- `VolumeServerConcurrentUploadLimit`
- `VolumeServerConcurrentDownloadLimit`
- `VolumeServerHandlerCounter` (labels: type — UploadLimitCond, DownloadLimitCond)
- Read/Write/Delete request counters
8.2. Metrics HTTP endpoint on `--metricsPort`.
8.3. Optional push-based metrics loop (`LoopPushingMetric`).
8.4. **Test:** Verify metric names and labels match Go output.
---
### Phase 9: HTTP Server & Handlers
**Goal:** All HTTP endpoints with exact same behavior as Go.
**Source files to port:**
- `weed/server/volume_server.go` → `src/server/volume_server.rs`
- `weed/server/volume_server_handlers.go` → `src/server/http_handlers.rs`
- `weed/server/volume_server_handlers_read.go` → `src/server/http_read.rs`
- `weed/server/volume_server_handlers_write.go` → `src/server/http_write.rs`
- `weed/server/volume_server_handlers_admin.go` → `src/server/http_admin.rs`
- `weed/server/volume_server_handlers_helper.go` (URL parsing, proxy, JSON responses)
- `weed/server/volume_server_handlers_ui.go` → `src/server/http_admin.rs`
**Steps:**
9.1. **URL path parsing** — from `handlers_helper.go`:
- Parse `/<vid>,<fid>` and `/<vid>/<fid>` patterns
- Extract volume ID, file ID, filename, ext
9.2. **Route dispatch** — from `privateStoreHandler` and `publicReadOnlyHandler`:
- `GET /` → `GetOrHeadHandler`
- `HEAD /` → `GetOrHeadHandler`
- `POST /` → `PostHandler` (whitelist gated)
- `PUT /` → `PostHandler` (whitelist gated)
- `DELETE /` → `DeleteHandler` (whitelist gated)
- `OPTIONS /` → CORS preflight
- `GET /status` → JSON status
- `GET /healthz` → health check
- `GET /ui/index.html` → HTML UI page
- Static resources (CSS/JS for UI)
9.3. **GET/HEAD handler** (`http_read.rs`) — from `handlers_read.go` (468 lines):
- JWT read authorization check
- Lookup needle by volume ID + needle ID + cookie
- ETag / If-None-Match / If-Modified-Since conditional responses
- Content-Type from stored MIME or filename extension
- Content-Disposition header
- Content-Encoding (gzip/zstd stored data)
- Range request support (HTTP 206 Partial Content)
- JPEG orientation fix (if configured)
- Proxy to replica on local miss (readMode=proxy)
- Redirect to replica (readMode=redirect)
- Download tracking (in-flight size accounting)
9.4. **POST/PUT handler** (`http_write.rs`) — from `handlers_write.go` (170 lines):
- JWT write authorization check
- Multipart form parsing
- Extract file data, filename, content type, TTL, last-modified
- Optional gzip/zstd compression
- Write needle to volume
- Replicate to peers (same logic as Go's `DistributedOperation`)
- Return JSON: {name, size, eTag, error}
9.5. **DELETE handler** — already in handlers.go:
- JWT authorization
- Delete from local volume
- Replicate delete to peers
- Return JSON result
9.6. **Admin handlers** (`http_admin.rs`):
- `/status` → JSON with volumes, version, disk status
- `/healthz` → 200 OK if serving
- `/ui/index.html` → HTML dashboard
9.7. **Concurrency limiting** — from `handlers.go`:
- Upload concurrency limit with `sync::Condvar` + timeout
- Download concurrency limit with proxy fallback to replicas
- HTTP 429 on timeout, 499 on client cancel
- Replication traffic bypasses upload limits
9.8. **Public port** — if configured, separate listener with read-only routes (GET/HEAD/OPTIONS only).
9.9. **Request ID middleware** — generate unique request ID per request.
9.10. **Tests:**
- Integration: start server → upload file via POST → GET it back → verify content
- Integration: upload → DELETE → GET returns 404
- Integration: conditional GET with ETag → 304
- Integration: range request → 206 with correct bytes
- Integration: exceed upload limit → 429
- Integration: whitelist enforcement
- Integration: JWT enforcement
---
### Phase 10: gRPC Service Implementation
**Goal:** All 40 gRPC methods with exact logic.
**Source files to port:**
- `weed/server/volume_grpc_admin.go` (380 lines)
- `weed/server/volume_grpc_vacuum.go` (124 lines)
- `weed/server/volume_grpc_copy.go` (636 lines)
- `weed/server/volume_grpc_copy_incremental.go` (66 lines)
- `weed/server/volume_grpc_read_write.go` (74 lines)
- `weed/server/volume_grpc_batch_delete.go` (124 lines)
- `weed/server/volume_grpc_tail.go` (140 lines)
- `weed/server/volume_grpc_erasure_coding.go` (619 lines)
- `weed/server/volume_grpc_scrub.go` (121 lines)
- `weed/server/volume_grpc_tier_upload.go` (98 lines)
- `weed/server/volume_grpc_tier_download.go` (85 lines)
- `weed/server/volume_grpc_remote.go` (95 lines)
- `weed/server/volume_grpc_query.go` (69 lines)
- `weed/server/volume_grpc_state.go` (26 lines)
- `weed/server/volume_grpc_read_all.go` (35 lines)
- `weed/server/volume_grpc_client_to_master.go` (325 lines)
**Steps (grouped by functional area):**
10.1. **Implement `tonic::Service` for `VolumeServer`** — the generated trait from proto.
10.2. **Admin RPCs** (`grpc_admin.rs`):
- `AllocateVolume` — create volume on appropriate disk location
- `VolumeMount` / `VolumeUnmount` / `VolumeDelete`
- `VolumeMarkReadonly` / `VolumeMarkWritable`
- `VolumeConfigure` — change replication
- `VolumeStatus` — return read-only, size, file counts
- `VolumeServerStatus` — disk statuses, memory, version, DC, rack
- `VolumeServerLeave` — deregister from master
- `DeleteCollection`
- `VolumeNeedleStatus` — get needle metadata by ID
- `Ping` — latency measurement
- `GetState` / `SetState` — maintenance mode
10.3. **Vacuum RPCs** (`grpc_vacuum.rs`):
- `VacuumVolumeCheck` — return garbage ratio
- `VacuumVolumeCompact` — stream progress (streaming response)
- `VacuumVolumeCommit` — finalize compaction
- `VacuumVolumeCleanup` — remove temp files
10.4. **Copy RPCs** (`grpc_copy.rs`):
- `VolumeCopy` — stream .dat/.idx from source to create local copy
- `VolumeSyncStatus` — return sync metadata
- `VolumeIncrementalCopy` — stream .dat delta since timestamp (streaming)
- `CopyFile` — generic file copy by extension (streaming)
- `ReceiveFile` — receive streamed file (client streaming)
- `ReadVolumeFileStatus` — return file timestamps and sizes
10.5. **Read/Write RPCs** (`grpc_read_write.rs`):
- `ReadNeedleBlob` — raw needle blob read
- `ReadNeedleMeta` — needle metadata
- `WriteNeedleBlob` — raw needle blob write
- `ReadAllNeedles` — stream all needles from volume(s) (streaming)
10.6. **Batch delete** (`grpc_batch_delete.rs`):
- `BatchDelete` — delete multiple file IDs, return per-ID results
10.7. **Tail RPCs** (`grpc_tail.rs`):
- `VolumeTailSender` — stream new needles since timestamp (streaming)
- `VolumeTailReceiver` — connect to another volume server and tail its changes
10.8. **Erasure coding RPCs** (`grpc_erasure_coding.rs`):
- `VolumeEcShardsGenerate` — generate EC shards from volume
- `VolumeEcShardsRebuild` — rebuild missing shards
- `VolumeEcShardsCopy` — copy shards from another server
- `VolumeEcShardsDelete` — delete EC shards
- `VolumeEcShardsMount` / `VolumeEcShardsUnmount`
- `VolumeEcShardRead` — read from EC shard (streaming)
- `VolumeEcBlobDelete` — mark blob deleted in EC volume
- `VolumeEcShardsToVolume` — reconstruct volume from EC shards
- `VolumeEcShardsInfo` — return shard metadata
10.9. **Scrub RPCs** (`grpc_scrub.rs`):
- `ScrubVolume` — integrity check volumes (INDEX / FULL / LOCAL modes)
- `ScrubEcVolume` — integrity check EC volumes
10.10. **Tier RPCs** (`grpc_tier.rs`):
- `VolumeTierMoveDatToRemote` — upload to remote backend (streaming progress)
- `VolumeTierMoveDatFromRemote` — download from remote (streaming progress)
10.11. **Remote storage** (`grpc_remote.rs`):
- `FetchAndWriteNeedle` — fetch from remote storage, write locally, replicate
10.12. **Query** (`grpc_query.rs`):
- `Query` — experimental CSV/JSON/Parquet select on stored data (streaming)
10.13. **Master heartbeat** (`grpc_client_to_master.rs`):
- `heartbeat()` background task — periodic gRPC stream to master
- Send: volume info, EC shard info, disk stats, has-no-space flags, deleted volumes
- Receive: volume size limit, leader address, metrics config
- Reconnect on failure with backoff
- `StopHeartbeat()` for graceful shutdown
10.14. **Tests:**
- Integration test per RPC: call via tonic client → verify response
- Streaming RPCs: verify all chunks received
- Error cases: invalid volume ID, non-existent volume, etc.
- Heartbeat: mock master gRPC server, verify registration
---
### Phase 11: Startup, Lifecycle & Graceful Shutdown
**Goal:** Full server startup matching Go's `runVolume()` and `startVolumeServer()`.
**Steps:**
11.1. **Startup sequence** (match `volume.go` exactly):
1. Load security configuration from `security.toml`
2. Start metrics server on metrics port
3. Parse folder/max/minFreeSpace/diskType/tags
4. Validate all directory writable
5. Resolve IP, bind IP, public URL, gRPC port
6. Create `VolumeServer` struct
7. Check with master (initial handshake)
8. Create `Store` (loads all existing volumes from disk)
9. Create security `Guard`
10. Register HTTP routes on admin mux
11. Optionally register public mux
12. Start gRPC server on gRPC port
13. Start public HTTP server (if separated)
14. Start cluster HTTP server (with optional TLS)
15. Start heartbeat background task
16. Start metrics push loop
17. Register SIGHUP handler for config reload + new volume loading
11.2. **Graceful shutdown** (match Go exactly):
1. On SIGINT/SIGTERM:
2. Stop heartbeat (notify master we're leaving)
3. Wait `preStopSeconds`
4. Stop public HTTP server
5. Stop cluster HTTP server
6. Graceful stop gRPC server
7. `volumeServer.Shutdown()` → `store.Close()` (flush all volumes)
11.3. **Reload** (SIGHUP):
- Reload security config
- Update whitelist
- Load newly appeared volumes from disk
11.4. **Tests:**
- Start server → send SIGTERM → verify clean shutdown
- Start server → SIGHUP → verify config reloaded
---
### Phase 12: Integration & Cross-Compatibility Testing
**Goal:** Rust volume server is a drop-in replacement for Go volume server.
**Steps:**
12.1. **Binary compatibility tests:**
- Create volumes with Go volume server
- Start Rust volume server on same data directory
- Read all data → verify identical
- Write new data with Rust → read with Go → verify
12.2. **API compatibility tests:**
- Run same HTTP requests against both Go and Rust servers
- Compare response bodies, headers, status codes
- Test all gRPC RPCs against both
12.3. **Master interop test:**
- Start Go master server
- Register Rust volume server
- Verify heartbeat works
- Verify volume assignment works
- Upload via filer → stored on Rust volume server → read back
12.4. **Performance benchmarks:**
- Throughput: sequential writes, sequential reads
- Latency: p50/p99 for read/write
- Concurrency: parallel reads/writes
- Compare Rust vs Go numbers
12.5. **Edge cases:**
- Volume at max size
- Disk full handling
- Corrupt .dat file recovery
- Network partition during replication
- EC shard loss + rebuild
---
## Execution Order & Dependencies
```
Phase 1 (Skeleton + CLI) ← no deps, start here
↓
Phase 2 (Storage types) ← needs Phase 1 (types used everywhere)
↓
Phase 3 (Volume struct) ← needs Phase 2
↓
Phase 4 (Store manager) ← needs Phase 3
↓
Phase 7 (Security) ← independent, can parallel with 3-4
Phase 8 (Metrics) ← independent, can parallel with 3-4
↓
Phase 9 (HTTP server) ← needs Phase 4 + 7 + 8
Phase 10 (gRPC server) ← needs Phase 4 + 7 + 8
↓
Phase 5 (Erasure coding) ← needs Phase 4, wire into Phase 10
Phase 6 (Tiered storage) ← needs Phase 4, wire into Phase 10
↓
Phase 11 (Startup + shutdown) ← needs Phase 9 + 10
↓
Phase 12 (Integration tests) ← needs all above
```
## Estimated Scope
| Phase | Estimated Rust Lines | Complexity |
|---|---|---|
| 1. Skeleton + CLI | ~400 | Low |
| 2. Storage types | ~2,000 | High (binary compat critical) |
| 3. Volume struct | ~2,500 | High |
| 4. Store manager | ~1,000 | Medium |
| 5. Erasure coding | ~3,000 | High |
| 6. Tiered storage | ~1,500 | Medium |
| 7. Security | ~500 | Medium |
| 8. Metrics | ~300 | Low |
| 9. HTTP server | ~2,000 | High |
| 10. gRPC server | ~3,500 | High |
| 11. Startup/shutdown | ~500 | Medium |
| 12. Integration tests | ~2,000 | Medium |
| **Total** | **~19,000** | |
## Critical Invariants to Preserve
1. **Binary format compatibility** — Rust must read/write `.dat`, `.idx`, `.vif`, `.ecX` files identically to Go. A single byte off = data loss.
2. **gRPC wire compatibility** — Same proto, same field semantics. Go master must talk to Rust volume server seamlessly.
3. **HTTP API compatibility** — Same URL patterns, same JSON response shapes, same headers, same status codes.
4. **Replication protocol** — Write replication between Go and Rust volume servers must work bidirectionally.
5. **Heartbeat protocol** — Rust volume server must register with Go master and maintain heartbeat.
6. **CRC32 algorithm** — Must use IEEE polynomial (same as Go's `crc32.ChecksumIEEE`).
7. **JWT compatibility** — Tokens signed by Go filer/master must be verifiable by Rust volume server and vice versa.
-2
View File
@@ -7,8 +7,6 @@
- [Evercam Camera Management Software](https://evercam.io/)
- [Spherical Elephant GmbH](https://www.sphericalelephant.com)
- [WizardTales GmbH](https://www.wizardtales.com)
- [Nimbus Web Services](https://nimbusws.com)
- <h2 align="center">Backers</h2>
-419
View File
@@ -1,419 +0,0 @@
package main
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/seaweedfs/seaweedfs/weed/server/postgres"
"github.com/seaweedfs/seaweedfs/weed/util"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
)
const usageLine = "weed-db -port=5432 -master=<master_server>"
const longHelp = `Start a PostgreSQL wire protocol compatible database server that provides SQL query access to SeaweedFS.
This database server enables any PostgreSQL client, tool, or application to connect to SeaweedFS
and execute SQL queries against MQ topics. It implements the PostgreSQL wire protocol for maximum
compatibility with the existing PostgreSQL ecosystem.
Examples:
# Start database server on default port 5432
weed-db
# Start with MD5 authentication using JSON format (recommended)
weed-db -auth=md5 -users='{"admin":"secret","readonly":"view123"}'
# Start with complex passwords using JSON format
weed-db -auth=md5 -users='{"admin":"pass;with;semicolons","user":"password:with:colons"}'
# Start with credentials from JSON file (most secure)
weed-db -auth=md5 -users="@/etc/seaweedfs/users.json"
# Start with custom port and master
weed-db -port=5433 -master=master1:9333
# Allow connections from any host
weed-db -host=0.0.0.0 -port=5432
# Start with TLS encryption
weed-db -tls-cert=server.crt -tls-key=server.key
Client Connection Examples:
# psql command line client
psql "host=localhost port=5432 dbname=default user=seaweedfs"
psql -h localhost -p 5432 -U seaweedfs -d default
# With password
PGPASSWORD=secret psql -h localhost -p 5432 -U admin -d default
# Connection string
psql "postgresql://admin:secret@localhost:5432/default"
Programming Language Examples:
# Python (psycopg2)
import psycopg2
conn = psycopg2.connect(
host="localhost", port=5432,
user="seaweedfs", database="default"
)
# Java JDBC
String url = "jdbc:postgresql://localhost:5432/default";
Connection conn = DriverManager.getConnection(url, "seaweedfs", "");
# Go (lib/pq)
db, err := sql.Open("postgres", "host=localhost port=5432 user=seaweedfs dbname=default sslmode=disable")
# Node.js (pg)
const client = new Client({
host: 'localhost', port: 5432,
user: 'seaweedfs', database: 'default'
});
Supported SQL Operations:
- SELECT queries on MQ topics
- DESCRIBE/DESC table_name commands
- EXPLAIN query execution plans
- SHOW DATABASES/TABLES commands
- Aggregation functions (COUNT, SUM, AVG, MIN, MAX)
- WHERE clauses with filtering
- System columns (_timestamp_ns, _key, _source)
- Basic PostgreSQL system queries (version(), current_database(), current_user)
Authentication Methods:
- trust: No authentication required (default)
- password: Clear text password authentication
- md5: MD5 password authentication
User Credential Formats:
- JSON format: '{"user1":"pass1","user2":"pass2"}' (supports any special characters)
- File format: "@/path/to/users.json" (JSON file)
Note: JSON format supports passwords with semicolons, colons, and any other special characters.
File format is recommended for production to keep credentials secure.
Compatible Tools:
- psql (PostgreSQL command line client)
- Any PostgreSQL JDBC/ODBC compatible tool
Security Features:
- Multiple authentication methods
- TLS encryption support
- Read-only access (no data modification)
Performance Features:
- Fast path aggregation optimization (COUNT, MIN, MAX without WHERE clauses)
- Hybrid data scanning (parquet files + live logs)
- PostgreSQL wire protocol
- Query result streaming
`
type Options struct {
Host string
Port int
MasterAddr string
AuthMethod string
Users string
Database string
MaxConns int
IdleTimeout string
TLSCert string
TLSKey string
}
// Run executes the weed-db CLI.
func Run(args []string) int {
fs := flag.NewFlagSet("weed-db", flag.ContinueOnError)
usageWriter := io.Writer(os.Stderr)
fs.SetOutput(usageWriter)
var opts Options
fs.StringVar(&opts.Host, "host", "localhost", "Database server host")
fs.IntVar(&opts.Port, "port", 5432, "Database server port")
fs.StringVar(&opts.MasterAddr, "master", "localhost:9333", "SeaweedFS master server address")
fs.StringVar(&opts.AuthMethod, "auth", "trust", "Authentication method: trust, password, md5")
fs.StringVar(&opts.Users, "users", "", "User credentials for auth (JSON format '{\"user1\":\"pass1\",\"user2\":\"pass2\"}' or file '@/path/to/users.json')")
fs.StringVar(&opts.Database, "database", "default", "Default database name")
fs.IntVar(&opts.MaxConns, "max-connections", 100, "Maximum concurrent connections per server")
fs.StringVar(&opts.IdleTimeout, "idle-timeout", "1h", "Connection idle timeout")
fs.StringVar(&opts.TLSCert, "tls-cert", "", "TLS certificate file path")
fs.StringVar(&opts.TLSKey, "tls-key", "", "TLS private key file path")
fs.Usage = func() {
fmt.Fprintf(usageWriter, "Usage: %s\n\n%s\n", usageLine, longHelp)
fmt.Fprintln(usageWriter, "Default Parameters:")
fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
return 2
}
if !runWithOptions(&opts) {
return 1
}
return 0
}
func runWithOptions(opts *Options) bool {
util.LoadConfiguration("security", false)
// Validate options.
if opts.MasterAddr == "" {
fmt.Fprintf(os.Stderr, "Error: master address is required\n")
return false
}
// Parse authentication method.
authMethod, err := parseAuthMethod(opts.AuthMethod)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
return false
}
// Parse user credentials.
users, err := parseUsers(opts.Users, authMethod)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
return false
}
// Parse idle timeout.
idleTimeout, err := time.ParseDuration(opts.IdleTimeout)
if err != nil {
fmt.Fprintf(os.Stderr, "Error parsing idle timeout: %v\n", err)
return false
}
// Validate port number.
if err := validatePortNumber(opts.Port); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
return false
}
// Setup TLS if requested.
var tlsConfig *tls.Config
if opts.TLSCert != "" && opts.TLSKey != "" {
cert, err := tls.LoadX509KeyPair(opts.TLSCert, opts.TLSKey)
if err != nil {
fmt.Fprintf(os.Stderr, "Error loading TLS certificates: %v\n", err)
return false
}
tlsConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
}
}
// Create server configuration.
config := &postgres.PostgreSQLServerConfig{
Host: opts.Host,
Port: opts.Port,
AuthMethod: authMethod,
Users: users,
Database: opts.Database,
MaxConns: opts.MaxConns,
IdleTimeout: idleTimeout,
TLSConfig: tlsConfig,
}
// Create database server.
dbServer, err := postgres.NewPostgreSQLServer(config, opts.MasterAddr)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating database server: %v\n", err)
return false
}
// Print startup information.
fmt.Printf("Starting SeaweedFS Database Server...\n")
fmt.Printf("Host: %s\n", opts.Host)
fmt.Printf("Port: %d\n", opts.Port)
fmt.Printf("Master: %s\n", opts.MasterAddr)
fmt.Printf("Database: %s\n", opts.Database)
fmt.Printf("Auth Method: %s\n", opts.AuthMethod)
fmt.Printf("Max Connections: %d\n", opts.MaxConns)
fmt.Printf("Idle Timeout: %s\n", opts.IdleTimeout)
if tlsConfig != nil {
fmt.Printf("TLS: Enabled\n")
} else {
fmt.Printf("TLS: Disabled\n")
}
if len(users) > 0 {
fmt.Printf("Users: %d configured\n", len(users))
}
fmt.Printf("\nDatabase Connection Examples:\n")
fmt.Printf(" psql -h %s -p %d -U seaweedfs -d %s\n", opts.Host, opts.Port, opts.Database)
if len(users) > 0 {
// Show first user as example.
for username := range users {
fmt.Printf(" psql -h %s -p %d -U %s -d %s\n", opts.Host, opts.Port, username, opts.Database)
break
}
}
fmt.Printf(" postgresql://%s:%d/%s\n", opts.Host, opts.Port, opts.Database)
fmt.Printf("\nSupported Operations:\n")
fmt.Printf(" - SELECT queries on MQ topics\n")
fmt.Printf(" - DESCRIBE/DESC table_name\n")
fmt.Printf(" - EXPLAIN query execution plans\n")
fmt.Printf(" - SHOW DATABASES/TABLES\n")
fmt.Printf(" - Aggregations: COUNT, SUM, AVG, MIN, MAX\n")
fmt.Printf(" - System columns: _timestamp_ns, _key, _source\n")
fmt.Printf(" - Basic PostgreSQL system queries\n")
fmt.Printf("\nReady for database connections!\n\n")
// Start the server.
err = dbServer.Start()
if err != nil {
fmt.Fprintf(os.Stderr, "Error starting database server: %v\n", err)
return false
}
// Set up signal handling for graceful shutdown.
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
// Wait for shutdown signal.
<-sigChan
fmt.Printf("\nReceived shutdown signal, stopping database server...\n")
// Create context with timeout for graceful shutdown.
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Stop the server with timeout.
done := make(chan error, 1)
go func() {
done <- dbServer.Stop()
}()
select {
case err := <-done:
if err != nil {
fmt.Fprintf(os.Stderr, "Error stopping database server: %v\n", err)
return false
}
fmt.Printf("Database server stopped successfully\n")
case <-ctx.Done():
fmt.Fprintf(os.Stderr, "Timeout waiting for database server to stop\n")
return false
}
return true
}
// parseAuthMethod parses the authentication method string.
func parseAuthMethod(method string) (postgres.AuthMethod, error) {
switch strings.ToLower(method) {
case "trust":
return postgres.AuthTrust, nil
case "password":
return postgres.AuthPassword, nil
case "md5":
return postgres.AuthMD5, nil
default:
return postgres.AuthTrust, fmt.Errorf("unsupported auth method '%s'. Supported: trust, password, md5", method)
}
}
// parseUsers parses the user credentials string with support for secure formats only.
// Supported formats:
// 1. JSON format: {"username":"password","username2":"password2"}
// 2. File format: /path/to/users.json or @/path/to/users.json
func parseUsers(usersStr string, authMethod postgres.AuthMethod) (map[string]string, error) {
users := make(map[string]string)
if usersStr == "" {
// No users specified.
if authMethod != postgres.AuthTrust {
return nil, fmt.Errorf("users must be specified when auth method is not 'trust'")
}
return users, nil
}
// Trim whitespace.
usersStr = strings.TrimSpace(usersStr)
// Determine format and parse accordingly.
if strings.HasPrefix(usersStr, "{") && strings.HasSuffix(usersStr, "}") {
// JSON format.
return parseUsersJSON(usersStr, authMethod)
}
// Check if it's a file path (with or without @ prefix) before declaring invalid format.
filePath := strings.TrimPrefix(usersStr, "@")
if _, err := os.Stat(filePath); err == nil {
// File format.
return parseUsersFile(usersStr, authMethod) // Pass original string to preserve @ handling.
}
// Invalid format.
return nil, fmt.Errorf("invalid user credentials format. Use JSON format '{\"user\":\"pass\"}' or file format '@/path/to/users.json' or 'path/to/users.json'. Legacy semicolon-separated format is no longer supported")
}
// parseUsersJSON parses user credentials from JSON format.
func parseUsersJSON(jsonStr string, authMethod postgres.AuthMethod) (map[string]string, error) {
var users map[string]string
if err := json.Unmarshal([]byte(jsonStr), &users); err != nil {
return nil, fmt.Errorf("invalid JSON format for users: %v", err)
}
// Validate users.
for username, password := range users {
if username == "" {
return nil, fmt.Errorf("empty username in JSON user specification")
}
if authMethod != postgres.AuthTrust && password == "" {
return nil, fmt.Errorf("empty password for user '%s' with auth method", username)
}
}
return users, nil
}
// parseUsersFile parses user credentials from a JSON file.
func parseUsersFile(filePath string, authMethod postgres.AuthMethod) (map[string]string, error) {
// Remove @ prefix if present.
filePath = strings.TrimPrefix(filePath, "@")
// Read file content.
content, err := os.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("failed to read users file '%s': %v", filePath, err)
}
contentStr := strings.TrimSpace(string(content))
// File must contain JSON format.
if !strings.HasPrefix(contentStr, "{") || !strings.HasSuffix(contentStr, "}") {
return nil, fmt.Errorf("users file '%s' must contain JSON format: {\"user\":\"pass\"}. Legacy formats are no longer supported", filePath)
}
// Parse as JSON.
return parseUsersJSON(contentStr, authMethod)
}
// validatePortNumber validates that the port number is reasonable.
func validatePortNumber(port int) error {
if port < 1 || port > 65535 {
return fmt.Errorf("port number must be between 1 and 65535, got %d", port)
}
if port < 1024 {
fmt.Fprintf(os.Stderr, "Warning: port number %d may require root privileges\n", port)
}
return nil
}
-7
View File
@@ -1,7 +0,0 @@
package main
import "os"
func main() {
os.Exit(Run(os.Args[1:]))
}
-7
View File
@@ -1,7 +0,0 @@
package main
import "os"
func main() {
os.Exit(Run(os.Args[1:]))
}
-623
View File
@@ -1,623 +0,0 @@
package main
import (
"context"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"path"
"strings"
"time"
"github.com/peterh/liner"
"github.com/seaweedfs/seaweedfs/weed/query/engine"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/util/sqlutil"
)
const usageLine = "weed-sql [-master=localhost:9333] [-interactive] [-file=query.sql] [-output=table|json|csv] [-database=dbname] [-query=\"SQL\"]"
const longHelp = `Enhanced SQL interface for SeaweedFS Message Queue topics with multiple execution modes.
Execution Modes:
- Interactive shell (default): weed-sql -interactive
- Single query: weed-sql -query "SELECT * FROM user_events"
- Batch from file: weed-sql -file queries.sql
- Context switching: weed-sql -database analytics -interactive
Output Formats:
- table: ASCII table format (default for interactive)
- json: JSON format (default for non-interactive)
- csv: Comma-separated values
Features:
- Full WHERE clause support (=, <, >, <=, >=, !=, LIKE, IN)
- Advanced pattern matching with LIKE wildcards (%, _)
- Multi-value filtering with IN operator
- Real MQ namespace and topic discovery
- Database context switching
Examples:
weed-sql -interactive
weed-sql -query "SHOW DATABASES" -output json
weed-sql -file batch_queries.sql -output csv
weed-sql -database analytics -query "SELECT COUNT(*) FROM metrics"
weed-sql -master broker1:9333 -interactive
`
type Options struct {
Master string
Interactive bool
File string
Output string
Database string
Query string
}
// OutputFormat represents different output formatting options.
type OutputFormat string
const (
OutputTable OutputFormat = "table"
OutputJSON OutputFormat = "json"
OutputCSV OutputFormat = "csv"
)
// SQLContext holds the execution context for SQL operations.
type SQLContext struct {
engine *engine.SQLEngine
currentDatabase string
outputFormat OutputFormat
interactive bool
master string
}
// Run executes the weed-sql CLI.
func Run(args []string) int {
fs := flag.NewFlagSet("weed-sql", flag.ContinueOnError)
usageWriter := io.Writer(os.Stderr)
fs.SetOutput(usageWriter)
var opts Options
fs.StringVar(&opts.Master, "master", "localhost:9333", "SeaweedFS master server HTTP address")
fs.BoolVar(&opts.Interactive, "interactive", false, "start interactive shell mode")
fs.StringVar(&opts.File, "file", "", "execute SQL queries from file")
fs.StringVar(&opts.Output, "output", "", "output format: table, json, csv (auto-detected if not specified)")
fs.StringVar(&opts.Database, "database", "", "default database context")
fs.StringVar(&opts.Query, "query", "", "execute single SQL query")
fs.Usage = func() {
fmt.Fprintf(usageWriter, "Usage: %s\n\n%s\n", usageLine, longHelp)
fmt.Fprintln(usageWriter, "Default Parameters:")
fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
return 2
}
if !runWithOptions(&opts) {
return 1
}
return 0
}
func runWithOptions(opts *Options) bool {
// Initialize SQL engine with master address for service discovery.
sqlEngine := engine.NewSQLEngine(opts.Master)
// Determine execution mode and output format.
interactive := opts.Interactive || (opts.Query == "" && opts.File == "")
outputFormat := determineOutputFormat(opts.Output, interactive)
// Create SQL context.
ctx := &SQLContext{
engine: sqlEngine,
currentDatabase: opts.Database,
outputFormat: outputFormat,
interactive: interactive,
master: opts.Master,
}
// Set current database in SQL engine if specified via command line.
if opts.Database != "" {
ctx.engine.GetCatalog().SetCurrentDatabase(opts.Database)
}
// Execute based on mode.
switch {
case opts.Query != "":
// Single query mode.
return executeSingleQuery(ctx, opts.Query)
case opts.File != "":
// Batch file mode.
return executeFileQueries(ctx, opts.File)
default:
// Interactive mode.
return runInteractiveShell(ctx)
}
}
// determineOutputFormat selects the appropriate output format.
func determineOutputFormat(specified string, interactive bool) OutputFormat {
switch strings.ToLower(specified) {
case "table":
return OutputTable
case "json":
return OutputJSON
case "csv":
return OutputCSV
default:
// Auto-detect based on mode.
if interactive {
return OutputTable
}
return OutputJSON
}
}
// executeSingleQuery executes a single query and outputs the result.
func executeSingleQuery(ctx *SQLContext, query string) bool {
if ctx.outputFormat != OutputTable {
// Suppress banner for non-interactive output.
return executeAndDisplay(ctx, query, false)
}
fmt.Printf("Executing query against %s...\n", ctx.master)
return executeAndDisplay(ctx, query, true)
}
// executeFileQueries processes SQL queries from a file.
func executeFileQueries(ctx *SQLContext, filename string) bool {
content, err := os.ReadFile(filename)
if err != nil {
fmt.Printf("Error reading file %s: %v\n", filename, err)
return false
}
if ctx.outputFormat == OutputTable && ctx.interactive {
fmt.Printf("Executing queries from %s against %s...\n", filename, ctx.master)
}
// Split file content into individual queries (robust approach).
queries := sqlutil.SplitStatements(string(content))
for i, query := range queries {
query = strings.TrimSpace(query)
if query == "" {
continue
}
if ctx.outputFormat == OutputTable && len(queries) > 1 {
fmt.Printf("\n--- Query %d ---\n", i+1)
}
if !executeAndDisplay(ctx, query, ctx.outputFormat == OutputTable) {
return false
}
}
return true
}
// runInteractiveShell starts the enhanced interactive shell with readline support.
func runInteractiveShell(ctx *SQLContext) bool {
fmt.Println("SeaweedFS Enhanced SQL Interface")
fmt.Println("Type 'help;' for help, 'exit;' to quit")
fmt.Printf("Connected to master: %s\n", ctx.master)
if ctx.currentDatabase != "" {
fmt.Printf("Current database: %s\n", ctx.currentDatabase)
}
fmt.Println("Advanced WHERE operators supported: <=, >=, !=, LIKE, IN")
fmt.Println("Use up/down arrows for command history")
fmt.Println()
// Initialize liner for readline functionality.
line := liner.NewLiner()
defer line.Close()
// Handle Ctrl+C gracefully.
line.SetCtrlCAborts(true)
grace.OnInterrupt(func() {
line.Close()
})
// Load command history.
historyPath := path.Join(os.TempDir(), "weed-sql-history")
if f, err := os.Open(historyPath); err == nil {
line.ReadHistory(f)
f.Close()
}
// Save history on exit.
defer func() {
if f, err := os.Create(historyPath); err == nil {
line.WriteHistory(f)
f.Close()
}
}()
var queryBuffer strings.Builder
for {
// Show prompt with current database context.
var prompt string
if queryBuffer.Len() == 0 {
if ctx.currentDatabase != "" {
prompt = fmt.Sprintf("seaweedfs:%s> ", ctx.currentDatabase)
} else {
prompt = "seaweedfs> "
}
} else {
prompt = " -> " // Continuation prompt.
}
// Read line with readline support.
input, err := line.Prompt(prompt)
if err != nil {
if err == liner.ErrPromptAborted {
fmt.Println("Query cancelled")
queryBuffer.Reset()
continue
}
if err != io.EOF {
fmt.Printf("Input error: %v\n", err)
}
break
}
lineStr := strings.TrimSpace(input)
// Handle empty lines.
if lineStr == "" {
continue
}
// Accumulate lines in query buffer.
if queryBuffer.Len() > 0 {
queryBuffer.WriteString(" ")
}
queryBuffer.WriteString(lineStr)
// Check if we have a complete statement (ends with semicolon or special command).
fullQuery := strings.TrimSpace(queryBuffer.String())
isComplete := strings.HasSuffix(lineStr, ";") ||
isSpecialCommand(fullQuery)
if !isComplete {
continue // Continue reading more lines.
}
// Add completed command to history.
line.AppendHistory(fullQuery)
// Handle special commands (with or without semicolon).
cleanQuery := strings.TrimSuffix(fullQuery, ";")
cleanQuery = strings.TrimSpace(cleanQuery)
if cleanQuery == "exit" || cleanQuery == "quit" || cleanQuery == "\\q" {
fmt.Println("Goodbye!")
break
}
if cleanQuery == "help" {
showEnhancedHelp()
queryBuffer.Reset()
continue
}
// Handle database switching - use proper SQL parser instead of manual parsing.
if strings.HasPrefix(strings.ToUpper(cleanQuery), "USE ") {
// Execute USE statement through the SQL engine for proper parsing.
result, err := ctx.engine.ExecuteSQL(context.Background(), cleanQuery)
if err != nil {
fmt.Printf("Error: %v\n\n", err)
} else if result.Error != nil {
fmt.Printf("Error: %v\n\n", result.Error)
} else {
// Extract the database name from the result message for CLI context.
if len(result.Rows) > 0 && len(result.Rows[0]) > 0 {
message := result.Rows[0][0].ToString()
// Extract database name from "Database changed to: dbname".
if strings.HasPrefix(message, "Database changed to: ") {
ctx.currentDatabase = strings.TrimPrefix(message, "Database changed to: ")
}
fmt.Printf("%s\n\n", message)
}
}
queryBuffer.Reset()
continue
}
// Handle output format switching.
if strings.HasPrefix(strings.ToUpper(cleanQuery), "\\FORMAT ") {
format := strings.TrimSpace(strings.TrimPrefix(strings.ToUpper(cleanQuery), "\\FORMAT "))
switch format {
case "TABLE":
ctx.outputFormat = OutputTable
fmt.Println("Output format set to: table")
case "JSON":
ctx.outputFormat = OutputJSON
fmt.Println("Output format set to: json")
case "CSV":
ctx.outputFormat = OutputCSV
fmt.Println("Output format set to: csv")
default:
fmt.Printf("Invalid format: %s. Supported: table, json, csv\n", format)
}
queryBuffer.Reset()
continue
}
// Execute SQL query (without semicolon).
executeAndDisplay(ctx, cleanQuery, true)
// Reset buffer for next query.
queryBuffer.Reset()
}
return true
}
// isSpecialCommand checks if a command is a special command that doesn't require semicolon.
func isSpecialCommand(query string) bool {
cleanQuery := strings.TrimSuffix(strings.TrimSpace(query), ";")
cleanQuery = strings.ToLower(cleanQuery)
// Special commands that work with or without semicolon.
specialCommands := []string{
"exit", "quit", "\\q", "help",
}
for _, cmd := range specialCommands {
if cleanQuery == cmd {
return true
}
}
// Commands that are exactly specific commands (not just prefixes).
parts := strings.Fields(strings.ToUpper(cleanQuery))
if len(parts) == 0 {
return false
}
return (parts[0] == "USE" && len(parts) >= 2) ||
strings.HasPrefix(strings.ToUpper(cleanQuery), "\\FORMAT ")
}
// executeAndDisplay executes a query and displays the result in the specified format.
func executeAndDisplay(ctx *SQLContext, query string, showTiming bool) bool {
startTime := time.Now()
// Execute the query.
execCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := ctx.engine.ExecuteSQL(execCtx, query)
if err != nil {
if ctx.outputFormat == OutputJSON {
errorResult := map[string]interface{}{
"error": err.Error(),
"query": query,
}
jsonBytes, _ := json.MarshalIndent(errorResult, "", " ")
fmt.Println(string(jsonBytes))
} else {
fmt.Printf("Error: %v\n", err)
}
return false
}
if result.Error != nil {
if ctx.outputFormat == OutputJSON {
errorResult := map[string]interface{}{
"error": result.Error.Error(),
"query": query,
}
jsonBytes, _ := json.MarshalIndent(errorResult, "", " ")
fmt.Println(string(jsonBytes))
} else {
fmt.Printf("Query Error: %v\n", result.Error)
}
return false
}
// Display results in the specified format.
switch ctx.outputFormat {
case OutputTable:
displayTableResult(result)
case OutputJSON:
displayJSONResult(result)
case OutputCSV:
displayCSVResult(result)
}
// Show execution time for interactive/table mode.
// Only show timing if there are columns or if result is truly empty.
if showTiming && ctx.outputFormat == OutputTable && (len(result.Columns) > 0 || len(result.Rows) == 0) {
elapsed := time.Since(startTime)
fmt.Printf("\n(%d rows in set, %.3f sec)\n\n", len(result.Rows), elapsed.Seconds())
}
return true
}
// displayTableResult formats and displays query results in ASCII table format.
func displayTableResult(result *engine.QueryResult) {
if len(result.Columns) == 0 {
fmt.Println("Empty result set")
return
}
// Calculate column widths for formatting.
colWidths := make([]int, len(result.Columns))
for i, col := range result.Columns {
colWidths[i] = len(col)
}
// Check data for wider columns.
for _, row := range result.Rows {
for i, val := range row {
if i < len(colWidths) {
valStr := val.ToString()
if len(valStr) > colWidths[i] {
colWidths[i] = len(valStr)
}
}
}
}
// Print header separator.
fmt.Print("+")
for _, width := range colWidths {
fmt.Print(strings.Repeat("-", width+2) + "+")
}
fmt.Println()
// Print column headers.
fmt.Print("|")
for i, col := range result.Columns {
fmt.Printf(" %-*s |", colWidths[i], col)
}
fmt.Println()
// Print separator.
fmt.Print("+")
for _, width := range colWidths {
fmt.Print(strings.Repeat("-", width+2) + "+")
}
fmt.Println()
// Print data rows.
for _, row := range result.Rows {
fmt.Print("|")
for i, val := range row {
if i < len(colWidths) {
fmt.Printf(" %-*s |", colWidths[i], val.ToString())
}
}
fmt.Println()
}
// Print bottom separator.
fmt.Print("+")
for _, width := range colWidths {
fmt.Print(strings.Repeat("-", width+2) + "+")
}
fmt.Println()
}
// displayJSONResult outputs query results in JSON format.
func displayJSONResult(result *engine.QueryResult) {
// Convert result to JSON-friendly format.
jsonResult := map[string]interface{}{
"columns": result.Columns,
"rows": make([]map[string]interface{}, len(result.Rows)),
"count": len(result.Rows),
}
// Convert rows to JSON objects.
for i, row := range result.Rows {
rowObj := make(map[string]interface{})
for j, val := range row {
if j < len(result.Columns) {
rowObj[result.Columns[j]] = val.ToString()
}
}
jsonResult["rows"].([]map[string]interface{})[i] = rowObj
}
// Marshal and print JSON.
jsonBytes, err := json.MarshalIndent(jsonResult, "", " ")
if err != nil {
fmt.Printf("Error formatting JSON: %v\n", err)
return
}
fmt.Println(string(jsonBytes))
}
// displayCSVResult outputs query results in CSV format.
func displayCSVResult(result *engine.QueryResult) {
// Handle execution plan results specially to avoid CSV quoting issues.
if len(result.Columns) == 1 && result.Columns[0] == "Query Execution Plan" {
// For execution plans, output directly without CSV encoding to avoid quotes.
for _, row := range result.Rows {
if len(row) > 0 {
fmt.Println(row[0].ToString())
}
}
return
}
// Standard CSV output for regular query results.
writer := csv.NewWriter(os.Stdout)
defer writer.Flush()
// Write headers.
if err := writer.Write(result.Columns); err != nil {
fmt.Printf("Error writing CSV headers: %v\n", err)
return
}
// Write data rows.
for _, row := range result.Rows {
csvRow := make([]string, len(row))
for i, val := range row {
csvRow[i] = val.ToString()
}
if err := writer.Write(csvRow); err != nil {
fmt.Printf("Error writing CSV row: %v\n", err)
return
}
}
}
func showEnhancedHelp() {
fmt.Println(`SeaweedFS Enhanced SQL Interface Help:
METADATA OPERATIONS:
SHOW DATABASES; - List all MQ namespaces
SHOW TABLES; - List all topics in current namespace
SHOW TABLES FROM database; - List topics in specific namespace
DESCRIBE table_name; - Show table schema
ADVANCED QUERYING:
SELECT * FROM table_name; - Query all data
SELECT col1, col2 FROM table WHERE ...; - Column projection
SELECT * FROM table WHERE id <= 100; - Range filtering
SELECT * FROM table WHERE name LIKE 'admin%'; - Pattern matching
SELECT * FROM table WHERE status IN ('active', 'pending'); - Multi-value
SELECT COUNT(*), MAX(id), MIN(id) FROM ...; - Aggregation functions
QUERY ANALYSIS:
EXPLAIN SELECT ...; - Show hierarchical execution plan
(data sources, optimizations, timing)
DDL OPERATIONS:
CREATE TABLE topic (field1 INT, field2 STRING); - Create topic
Note: ALTER TABLE and DROP TABLE are not supported
SPECIAL COMMANDS:
USE database_name; - Switch database context
\format table|json|csv - Change output format
help; - Show this help
exit; or quit; or \q - Exit interface
EXTENDED WHERE OPERATORS:
=, <, >, <=, >= - Comparison operators
!=, <> - Not equal operators
LIKE 'pattern%' - Pattern matching (% = any chars, _ = single char)
IN (value1, value2, ...) - Multi-value matching
AND, OR - Logical operators
EXAMPLES:
SELECT * FROM user_events WHERE user_id >= 10 AND status != 'deleted';
SELECT username FROM users WHERE email LIKE '%@company.com';
SELECT * FROM logs WHERE level IN ('error', 'warning') AND timestamp >= '2023-01-01';
EXPLAIN SELECT MAX(id) FROM events; -- View execution plan
Current Status: Full WHERE clause support + Real MQ integration`)
}
+2 -20
View File
@@ -2,30 +2,12 @@ FROM ubuntu:22.04
LABEL author="Chris Lu"
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
# For testing purposes, running as root avoids permission complexities and dependency
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
apt-get -o Acquire::http::Timeout=15 update && \
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
--no-install-recommends \
--no-install-suggests \
curl \
fio \
fuse \
ca-certificates \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /tmp/* \
&& rm -rf /var/tmp/*
RUN apt-get update && apt-get install -y curl fio fuse
RUN mkdir -p /etc/seaweedfs /data/filerldb2
COPY ./weed /usr/bin/
COPY ./filer.toml /etc/seaweedfs/filer.toml
COPY ./entrypoint_e2e.sh /entrypoint.sh
COPY ./entrypoint.sh /entrypoint.sh
# volume server grpc port
EXPOSE 18080
-132
View File
@@ -1,132 +0,0 @@
FROM golang:1.25 AS builder
RUN apt-get update && \
apt-get install -y build-essential wget ca-certificates && \
rm -rf /var/lib/apt/lists/*
ARG FDB_VERSION=7.4.5
ENV FDB_VERSION=${FDB_VERSION}
ARG TARGETARCH
# Install FoundationDB client libraries with SHA256 checksum verification
# Known SHA256 checksums for FoundationDB client packages (verified 2025-01-19)
# To add checksums for new versions: run docker/get_fdb_checksum.sh <version> <arch>
RUN cd /tmp && \
case "${TARGETARCH}" in \
"amd64") FDB_ARCH="amd64"; PACKAGE_ARCH="amd64" ;; \
"arm64") FDB_ARCH="arm64"; PACKAGE_ARCH="aarch64" ;; \
*) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
esac && \
case "${FDB_VERSION}_${FDB_ARCH}" in \
"7.4.5_amd64") \
EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;; \
"7.4.5_arm64") \
EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
"7.3.43_amd64") \
EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;; \
"7.3.43_arm64") \
echo "ERROR: FoundationDB ${FDB_VERSION} does not publish arm64 client packages." >&2; \
echo "Please upgrade to 7.4.5+ when targeting arm64." >&2; \
exit 1 ;; \
*) \
echo "ERROR: No checksum available for FDB version ${FDB_VERSION} on ${FDB_ARCH}" >&2; \
echo "This is a security requirement. To add verification:" >&2; \
echo " 1. Run: docker/get_fdb_checksum.sh ${FDB_VERSION} ${FDB_ARCH}" >&2; \
echo " 2. Add the checksum to this Dockerfile" >&2; \
echo "Refusing to proceed without checksum verification." >&2; \
exit 1 ;; \
esac && \
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb" && \
wget --timeout=30 --tries=3 https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE} && \
echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c - || \
(echo "ERROR: Checksum verification failed for FoundationDB ${FDB_VERSION} (${FDB_ARCH})" >&2; \
echo "Expected: ${EXPECTED_SHA256}" >&2; \
echo "This indicates either a corrupted download or potential tampering." >&2; \
exit 1) && \
dpkg -i ${PACKAGE} && \
rm ${PACKAGE}
# Set up FoundationDB environment variables for CGO
ENV CGO_CFLAGS="-I/usr/include/foundationdb"
ENV CGO_LDFLAGS="-lfdb_c"
# build SeaweedFS sources; prefer local context but fall back to git clone if context only has docker files
ARG SOURCE_REF=master
WORKDIR /go/src/github.com/seaweedfs/seaweedfs
COPY . .
RUN set -euo pipefail && \
if [ ! -d weed ]; then \
echo "Local build context does not include SeaweedFS sources; cloning ${SOURCE_REF}" >&2; \
mkdir -p /tmp/local-context && cp -a /go/src/github.com/seaweedfs/seaweedfs/. /tmp/local-context && \
cd / && rm -rf /go/src/github.com/seaweedfs/seaweedfs && \
git clone --depth 1 --branch ${SOURCE_REF} https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs && \
cp -a /tmp/local-context/. /go/src/github.com/seaweedfs/seaweedfs/docker/ && \
rm -rf /tmp/local-context && \
cd /go/src/github.com/seaweedfs/seaweedfs; \
fi && \
cd weed \
&& COMMIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${COMMIT_SHA}" \
&& go install -tags "5BytesOffset foundationdb" -ldflags "${LDFLAGS}"
FROM debian:bookworm-slim AS final
LABEL author="Chris Lu"
# Install runtime dependencies first
RUN apt-get update && \
apt-get install -y --no-install-recommends \
ca-certificates \
curl \
fuse \
wget && \
rm -rf /var/lib/apt/lists/*
# Reuse FoundationDB artifacts installed during the build stage
COPY --from=builder /usr/lib/libfdb_c* /usr/lib/
COPY --from=builder /usr/lib/foundationdb /usr/lib/foundationdb
COPY --from=builder /usr/bin/fdb* /usr/bin/
RUN ldconfig
# Copy SeaweedFS binary and configuration
COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_foundationdb.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Create non-root user
RUN groupadd -g 1000 seaweed && \
useradd -u 1000 -g seaweed -s /bin/bash -m seaweed
# volume server gprc port
EXPOSE 18080
# volume server http port
EXPOSE 8080
# filer server gprc port
EXPOSE 18888
# filer server http port
EXPOSE 8888
# master server shared gprc port
EXPOSE 19333
# master server shared http port
EXPOSE 9333
# s3 server http port
EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
VOLUME /data
WORKDIR /data
# Switch to non-root user
USER seaweed
ENTRYPOINT ["/entrypoint.sh"]
+44
View File
@@ -0,0 +1,44 @@
FROM gcc:11 as builder
RUN mkdir -p /go/src/github.com/seaweedfs/
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG BRANCH=${BRANCH:-master}
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& apt-get update \
&& apt-get install -y golang-src \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
FROM alpine AS final
LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
RUN apk add fuse # for weed mount
# volume server gprc port
EXPOSE 18080
# volume server http port
EXPOSE 8080
# filer server gprc port
EXPOSE 18888
# filer server http port
EXPOSE 8888
# master server shared gprc port
EXPOSE 19333
# master server shared http port
EXPOSE 9333
# s3 server http port
EXPOSE 8333
# webdav server http port
EXPOSE 7333
RUN mkdir -p /data/filerldb2
VOLUME /data
WORKDIR /data
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+8 -60
View File
@@ -1,65 +1,21 @@
FROM golang:1.25-alpine AS builder
FROM golang:1.19-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/seaweedfs/
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG BRANCH=${BRANCH:-master}
# Clone with full history and all tags to ensure all commits are available
RUN git clone --no-single-branch --tags https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG TAGS
RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
(git checkout $BRANCH || \
(echo "Checkout failed, fetching all history..." && \
git fetch --all --tags --prune && \
git checkout $BRANCH) || \
(echo "ERROR: Branch/commit $BRANCH not found in repository" && \
echo "Available branches:" && git branch -a && exit 1))
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
# Rust volume server: use pre-built binary from CI when available (placed in
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build
# for non-native architectures.
FROM alpine:3.23 as rust_builder
ARG TARGETARCH
ARG TAGS
COPY weed-volume-prebuilt/ /prebuilt/
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
WORKDIR /build/seaweed-volume
RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
echo "Using pre-built Rust binary for ${TARGETARCH}" && \
cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
if [ "$TAGS" = "5BytesOffset" ]; then \
cargo build --release; \
else \
cargo build --release --no-default-features; \
fi && \
cp target/release/weed-volume /weed-volume; \
else \
echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
touch /weed-volume; \
fi
FROM alpine AS final
LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/
# Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms)
COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# FIPS 140-3 mode is ON by default (Go 1.24+)
# To disable: docker run -e GODEBUG=fips140=off ...
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
RUN apk add fuse # for weed mount
# volume server gprc port
EXPOSE 18080
@@ -78,19 +34,11 @@ EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data/filerldb2 && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
RUN mkdir -p /data/filerldb2
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+6 -19
View File
@@ -1,16 +1,11 @@
FROM alpine AS final
LABEL author="Chris Lu"
COPY ./weed /usr/bin/weed
RUN chmod +x /usr/bin/weed && ls -la /usr/bin/weed
COPY ./weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY ./filer.toml /etc/seaweedfs/filer.toml
COPY ./entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
RUN apk add fuse # for weed mount
RUN apk add curl # for health checks
# volume server grpc port
EXPOSE 18080
@@ -29,19 +24,11 @@ EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data/filerldb2 && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
RUN mkdir -p /data/filerldb2
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+5 -6
View File
@@ -1,17 +1,16 @@
FROM golang:1.25 AS builder
FROM golang:1.19-buster as builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ARG ROCKSDB_VERSION=v10.10.1
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
ENV ROCKSDB_VERSION v7.5.3
# build RocksDB
RUN cd /tmp && \
git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
cd rocksdb && \
PORTABLE=1 make -j"$(nproc)" static_lib && \
PORTABLE=1 make static_lib && \
make install-static
ENV CGO_CFLAGS="-I/tmp/rocksdb/include"
ENV CGO_LDFLAGS="-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
+11 -25
View File
@@ -1,28 +1,27 @@
FROM golang:1.25 AS builder
FROM golang:1.19-buster as builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ARG ROCKSDB_VERSION=v10.10.1
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
ENV ROCKSDB_VERSION v7.5.3
# build RocksDB
RUN cd /tmp && \
git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
cd rocksdb && \
PORTABLE=1 make -j"$(nproc)" static_lib && \
PORTABLE=1 make static_lib && \
make install-static
ENV CGO_CFLAGS="-I/tmp/rocksdb/include"
ENV CGO_LDFLAGS="-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
# build SeaweedFS
RUN mkdir -p /go/src/github.com/seaweedfs/
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG BRANCH=master
ARG BRANCH=${BRANCH:-master}
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
@@ -32,12 +31,7 @@ COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse snappy gflags curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
RUN apk add fuse snappy gflags
# volume server gprc port
EXPOSE 18080
@@ -56,20 +50,12 @@ EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data/filer_rocksdb && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
RUN mkdir -p /data/filer_rocksdb
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+5 -13
View File
@@ -5,7 +5,7 @@ RUN mkdir -p /go/src/github.com/seaweedfs/
ADD . /go/src/github.com/seaweedfs/seaweedfs
RUN ls -al /go/src/github.com/seaweedfs/ && \
cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
@@ -15,12 +15,7 @@ COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
RUN apk add fuse snappy gflags tmux
# volume server gprc port
EXPOSE 18080
@@ -39,15 +34,12 @@ EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data/filer_rocksdb && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
RUN mkdir -p /data/filer_rocksdb
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+1 -1
View File
@@ -25,7 +25,7 @@ ENV \
NOSETESTS_EXCLUDE="" \
NOSETESTS_ATTR="" \
NOSETESTS_OPTIONS="" \
S3TEST_CONF="/s3tests.conf"
S3TEST_CONF="/s3test.conf"
ENTRYPOINT ["/bin/bash", "-c"]
CMD ["sleep 30 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
-17
View File
@@ -1,17 +0,0 @@
FROM tarantool/tarantool:3.3.1 AS builder
# install dependencies
RUN apt update && \
apt install -y git unzip cmake tt=2.7.0
# init tt dir structure, create dir for app, create symlink
RUN tt init && \
mkdir app && \
ln -sfn ${PWD}/app/ ${PWD}/instances.enabled/app
# copy cluster configs
COPY tarantool /opt/tarantool/app
# build app
RUN tt build app
+30 -60
View File
@@ -5,36 +5,22 @@ all: gen
gen: dev
cgo ?= 0
ldflags_extra ?= -extldflags -static
binary:
export SWCOMMIT=$(shell git rev-parse --short HEAD)
export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(SWCOMMIT)"
cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w $(ldflags_extra) $(SWLDFLAGS)" -o weed_binary && mv weed_binary ../docker/weed
cd ../other/mq_client_example/agent_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_pub_record ../../../docker/
cd ../other/mq_client_example/agent_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_sub_record ../../../docker/
export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
# Race detector requires CGO and dynamic linking - don't use -static
binary_race: options = -race
binary_race: cgo = 1
binary_race: ldflags_extra =
binary_race: binary
build: binary
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
build_e2e: binary_race
docker buildx build \
--cache-from=type=local,src=/tmp/.buildx-cache \
--cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \
--load \
-t chrislusf/seaweedfs:e2e \
-f Dockerfile.e2e .
# Move cache to avoid growing cache size
rm -rf /tmp/.buildx-cache || true
mv /tmp/.buildx-cache-new /tmp/.buildx-cache || true
docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset,tarantool
go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset
docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build .
go_build_large_disk:
@@ -43,100 +29,84 @@ go_build_large_disk:
build_rocksdb_dev_env:
docker build --no-cache -t chrislusf/rocksdb_dev_env -f Dockerfile.rocksdb_dev_env .
build_rocksdb_local: build_rocksdb_dev_env
build_rocksdb_local:
cd .. ; docker build --no-cache -t chrislusf/seaweedfs:rocksdb_local -f docker/Dockerfile.rocksdb_large_local .
build_rocksdb:
docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
build_tarantool_dev_env:
docker build --no-cache -t chrislusf/tarantool_dev_env -f Dockerfile.tarantool.dev_env .
s3tests_build:
docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
dev: build
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_race: binary_race
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_tls: build certstrap
ENV_FILE="tls.env" docker compose -f compose/local-dev-compose.yml -p seaweedfs up
ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_mount: build
docker compose -f compose/local-mount-compose.yml -p seaweedfs up
docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
run_image: build
docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local
profile_mount: build
docker compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
k8s: build
docker compose -f compose/local-k8s-compose.yml -p seaweedfs up
docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
dev_registry: build
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
docker-compose -f compose/local-registry-compose.yml -p seaweedfs up
dev_replicate:
docker build --build-arg TAGS=gocdk --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.go_build .
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
dev_replicate: build
docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
dev_auditlog: build
docker compose -f compose/local-auditlog-compose.yml -p seaweedfs up
docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
dev_nextcloud: build
docker compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
docker-compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
cluster: build
docker compose -f compose/local-cluster-compose.yml -p seaweedfs up
docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
2clusters: build
docker compose -f compose/local-clusters-compose.yml -p seaweedfs up
docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
2mount: build
docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
filer_backup: build
docker compose -f compose/local-filer-backup-compose.yml -p seaweedfs up
docker-compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
hashicorp_raft: build
docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
s3tests: build s3tests_build
docker compose -f compose/local-s3tests-compose.yml -p seaweedfs up
brokers: build
docker compose -f compose/local-brokers-compose.yml -p seaweedfs up
agent: build
docker compose -f compose/local-mq-test.yml -p seaweedfs up
docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
filer_etcd: build
docker stack deploy -c compose/swarm-etcd.yml fs
test_etcd: build
docker compose -f compose/test-etcd-filer.yml -p seaweedfs up
docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
test_ydb: tags = ydb
test_ydb: build
docker compose -f compose/test-ydb-filer.yml -p seaweedfs up
test_tarantool: tags = tarantool
test_tarantool: build_tarantool_dev_env build
docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
export
docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up
clean:
rm ./weed
certstrap:
go install -v github.com/square/certstrap@latest
certstrap --depot-path compose/tls init --curve P-256 --passphrase "" --common-name "SeaweedFS CA" || true
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --domain localhost --common-name volume01.dev || true
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name master01.dev || true
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name filer01.dev || true
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name client01.dev || true
go get github.com/square/certstrap
certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true
+2 -12
View File
@@ -1,15 +1,5 @@
# Docker
## Compose V2
SeaweedFS now uses the `v2` syntax `docker compose`
If you rely on using Docker Compose as docker-compose (with a hyphen), you can set up Compose V2 to act as a drop-in replacement of the previous docker-compose. Refer to the [Installing Compose](https://docs.docker.com/compose/install/) section for detailed instructions on upgrading.
Confirm your system has docker compose v2 with a version check
```bash
$ docker compose version
Docker Compose version v2.10.2
```
## Try it out
@@ -17,7 +7,7 @@ Docker Compose version v2.10.2
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml
docker compose -f seaweedfs-compose.yml -p seaweedfs up
docker-compose -f seaweedfs-compose.yml -p seaweedfs up
```
@@ -27,7 +17,7 @@ docker compose -f seaweedfs-compose.yml -p seaweedfs up
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml
docker compose -f seaweedfs-dev-compose.yml -p seaweedfs up
docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
```
-18
View File
@@ -1,18 +0,0 @@
FROM alpine:latest
# Install required packages
RUN apk add --no-cache \
ca-certificates \
fuse \
curl \
jq
# Copy our locally built binary
COPY weed-local /usr/bin/weed
RUN chmod +x /usr/bin/weed
# Create working directory
WORKDIR /data
# Default command
ENTRYPOINT ["/usr/bin/weed"]
@@ -1,438 +0,0 @@
# SeaweedFS EC Worker Testing Environment
This Docker Compose setup provides a comprehensive testing environment for SeaweedFS Erasure Coding (EC) workers using **official SeaweedFS commands**.
## 📂 Directory Structure
The testing environment is located in `docker/admin_integration/` and includes:
```
docker/admin_integration/
├── Makefile # Main management interface
├── docker-compose-ec-test.yml # Docker compose configuration
├── EC-TESTING-README.md # This documentation
└── run-ec-test.sh # Quick start script
```
## 🏗️ Architecture
The testing environment uses **official SeaweedFS commands** and includes:
- **1 Master Server** (port 9333) - Coordinates the cluster with 50MB volume size limit
- **6 Volume Servers** (ports 8080-8085) - Distributed across 2 data centers and 3 racks for diversity
- **1 Filer** (port 8888) - Provides file system interface
- **1 Admin Server** (port 23646) - Detects volumes needing EC and manages workers using official `admin` command
- **3 EC Workers** - Execute erasure coding tasks using official `worker` command with task-specific working directories
- **1 Load Generator** - Continuously writes and deletes files using SeaweedFS shell commands
- **1 Monitor** - Tracks cluster health and EC progress using shell scripts
## ✨ New Features
### **Task-Specific Working Directories**
Each worker now creates dedicated subdirectories for different task types:
- `/work/erasure_coding/` - For EC encoding tasks
- `/work/vacuum/` - For vacuum cleanup tasks
- `/work/balance/` - For volume balancing tasks
This provides:
- **Organization**: Each task type gets isolated working space
- **Debugging**: Easy to find files/logs related to specific task types
- **Cleanup**: Can clean up task-specific artifacts easily
- **Concurrent Safety**: Different task types won't interfere with each other's files
## 🚀 Quick Start
### Prerequisites
- Docker and Docker Compose installed
- GNU Make installed
- At least 4GB RAM available for containers
- Ports 8080-8085, 8888, 9333, 23646 available
### Start the Environment
```bash
# Navigate to the admin integration directory
cd docker/admin_integration/
# Show available commands
make help
# Start the complete testing environment
make start
```
The `make start` command will:
1. Start all services using official SeaweedFS images
2. Configure workers with task-specific working directories
3. Wait for services to be ready
4. Display monitoring URLs and run health checks
### Alternative Commands
```bash
# Quick start aliases
make up # Same as 'make start'
# Development mode (higher load for faster testing)
make dev-start
# Build images without starting
make build
```
## 📋 Available Make Targets
Run `make help` to see all available targets:
### **🚀 Main Operations**
- `make start` - Start the complete EC testing environment
- `make stop` - Stop all services
- `make restart` - Restart all services
- `make clean` - Complete cleanup (containers, volumes, images)
### **📊 Monitoring & Status**
- `make health` - Check health of all services
- `make status` - Show status of all containers
- `make urls` - Display all monitoring URLs
- `make monitor` - Open monitor dashboard in browser
- `make monitor-status` - Show monitor status via API
- `make volume-status` - Show volume status from master
- `make admin-status` - Show admin server status
- `make cluster-status` - Show complete cluster status
### **📋 Logs Management**
- `make logs` - Show logs from all services
- `make logs-admin` - Show admin server logs
- `make logs-workers` - Show all worker logs
- `make logs-worker1/2/3` - Show specific worker logs
- `make logs-load` - Show load generator logs
- `make logs-monitor` - Show monitor logs
- `make backup-logs` - Backup all logs to files
### **⚖️ Scaling & Testing**
- `make scale-workers WORKERS=5` - Scale workers to 5 instances
- `make scale-load RATE=25` - Increase load generation rate
- `make test-ec` - Run focused EC test scenario
### **🔧 Development & Debug**
- `make shell-admin` - Open shell in admin container
- `make shell-worker1` - Open shell in worker container
- `make debug` - Show debug information
- `make troubleshoot` - Run troubleshooting checks
## 📊 Monitoring URLs
| Service | URL | Description |
|---------|-----|-------------|
| Master UI | http://localhost:9333 | Cluster status and topology |
| Filer | http://localhost:8888 | File operations |
| Admin Server | http://localhost:23646/ | Task management |
| Monitor | http://localhost:9999/status | Complete cluster monitoring |
| Volume Servers | http://localhost:8080-8085/status | Individual volume server stats |
Quick access: `make urls` or `make monitor`
## 🔄 How EC Testing Works
### 1. Continuous Load Generation
- **Write Rate**: 10 files/second (1-5MB each)
- **Delete Rate**: 2 files/second
- **Target**: Fill volumes to 50MB limit quickly
### 2. Volume Detection
- Admin server scans master every 30 seconds
- Identifies volumes >40MB (80% of 50MB limit)
- Queues EC tasks for eligible volumes
### 3. EC Worker Assignment
- **Worker 1**: EC specialist (max 2 concurrent tasks)
- **Worker 2**: EC + Vacuum hybrid (max 2 concurrent tasks)
- **Worker 3**: EC + Vacuum hybrid (max 1 concurrent task)
### 4. Comprehensive EC Process
Each EC task follows 6 phases:
1. **Copy Volume Data** (5-15%) - Stream .dat/.idx files locally
2. **Mark Read-Only** (20-25%) - Ensure data consistency
3. **Local Encoding** (30-60%) - Create 14 shards (10+4 Reed-Solomon)
4. **Calculate Placement** (65-70%) - Smart rack-aware distribution
5. **Distribute Shards** (75-90%) - Upload to optimal servers
6. **Verify & Cleanup** (95-100%) - Validate and clean temporary files
### 5. Real-Time Monitoring
- Volume analysis and EC candidate detection
- Worker health and task progress
- No data loss verification
- Performance metrics
## 📋 Key Features Tested
### ✅ EC Implementation Features
- [x] Local volume data copying with progress tracking
- [x] Local Reed-Solomon encoding (10+4 shards)
- [x] Intelligent shard placement with rack awareness
- [x] Load balancing across available servers
- [x] Backup server selection for redundancy
- [x] Detailed step-by-step progress tracking
- [x] Comprehensive error handling and recovery
### ✅ Infrastructure Features
- [x] Multi-datacenter topology (dc1, dc2)
- [x] Rack diversity (rack1, rack2, rack3)
- [x] Volume size limits (50MB)
- [x] Worker capability matching
- [x] Health monitoring and alerting
- [x] Continuous workload simulation
## 🛠️ Common Usage Patterns
### Basic Testing Workflow
```bash
# Start environment
make start
# Watch progress
make monitor-status
# Check for EC candidates
make volume-status
# View worker activity
make logs-workers
# Stop when done
make stop
```
### High-Load Testing
```bash
# Start with higher load
make dev-start
# Scale up workers and load
make scale-workers WORKERS=5
make scale-load RATE=50
# Monitor intensive EC activity
make logs-admin
```
### Debugging Issues
```bash
# Check port conflicts and system state
make troubleshoot
# View specific service logs
make logs-admin
make logs-worker1
# Get shell access for debugging
make shell-admin
make shell-worker1
# Check detailed status
make debug
```
### Development Iteration
```bash
# Quick restart after code changes
make restart
# Rebuild and restart
make clean
make start
# Monitor specific components
make logs-monitor
```
## 📈 Expected Results
### Successful EC Testing Shows:
1. **Volume Growth**: Steady increase in volume sizes toward 50MB limit
2. **EC Detection**: Admin server identifies volumes >40MB for EC
3. **Task Assignment**: Workers receive and execute EC tasks
4. **Shard Distribution**: 14 shards distributed across 6 volume servers
5. **No Data Loss**: All files remain accessible during and after EC
6. **Performance**: EC tasks complete within estimated timeframes
### Sample Monitor Output:
```bash
# Check current status
make monitor-status
# Output example:
{
"monitor": {
"uptime": "15m30s",
"master_addr": "master:9333",
"admin_addr": "admin:9900"
},
"stats": {
"VolumeCount": 12,
"ECTasksDetected": 3,
"WorkersActive": 3
}
}
```
## 🔧 Configuration
### Environment Variables
You can customize the environment by setting variables:
```bash
# High load testing
WRITE_RATE=25 DELETE_RATE=5 make start
# Extended test duration
TEST_DURATION=7200 make start # 2 hours
```
### Scaling Examples
```bash
# Scale workers
make scale-workers WORKERS=6
# Increase load generation
make scale-load RATE=30
# Combined scaling
make scale-workers WORKERS=4
make scale-load RATE=40
```
## 🧹 Cleanup Options
```bash
# Stop services only
make stop
# Remove containers but keep volumes
make down
# Remove data volumes only
make clean-volumes
# Remove built images only
make clean-images
# Complete cleanup (everything)
make clean
```
## 🐛 Troubleshooting
### Quick Diagnostics
```bash
# Run complete troubleshooting
make troubleshoot
# Check specific components
make health
make debug
make status
```
### Common Issues
**Services not starting:**
```bash
# Check port availability
make troubleshoot
# View startup logs
make logs-master
make logs-admin
```
**No EC tasks being created:**
```bash
# Check volume status
make volume-status
# Increase load to fill volumes faster
make scale-load RATE=30
# Check admin detection
make logs-admin
```
**Workers not responding:**
```bash
# Check worker registration
make admin-status
# View worker logs
make logs-workers
# Restart workers
make restart
```
### Performance Tuning
**For faster testing:**
```bash
make dev-start # Higher default load
make scale-load RATE=50 # Very high load
```
**For stress testing:**
```bash
make scale-workers WORKERS=8
make scale-load RATE=100
```
## 📚 Technical Details
### Network Architecture
- Custom bridge network (172.20.0.0/16)
- Service discovery via container names
- Health checks for all services
### Storage Layout
- Each volume server: max 100 volumes
- Data centers: dc1, dc2
- Racks: rack1, rack2, rack3
- Volume limit: 50MB per volume
### EC Algorithm
- Reed-Solomon RS(10,4)
- 10 data shards + 4 parity shards
- Rack-aware distribution
- Backup server redundancy
### Make Integration
- Color-coded output for better readability
- Comprehensive help system (`make help`)
- Parallel execution support
- Error handling and cleanup
- Cross-platform compatibility
## 🎯 Quick Reference
```bash
# Essential commands
make help # Show all available targets
make start # Start complete environment
make health # Check all services
make monitor # Open dashboard
make logs-admin # View admin activity
make clean # Complete cleanup
# Monitoring
make volume-status # Check for EC candidates
make admin-status # Check task queue
make monitor-status # Full cluster status
# Scaling & Testing
make test-ec # Run focused EC test
make scale-load RATE=X # Increase load
make troubleshoot # Diagnose issues
```
This environment provides a realistic testing scenario for SeaweedFS EC workers with actual data operations, comprehensive monitoring, and easy management through Make targets.
-346
View File
@@ -1,346 +0,0 @@
# SeaweedFS Admin Integration Test Makefile
# Tests the admin server and worker functionality using official weed commands
.PHONY: help build build-and-restart restart-workers start stop restart logs clean status test admin-ui worker-logs master-logs admin-logs vacuum-test vacuum-demo vacuum-status vacuum-data vacuum-data-high vacuum-data-low vacuum-continuous vacuum-clean vacuum-help
.DEFAULT_GOAL := help
COMPOSE_FILE := docker-compose-ec-test.yml
PROJECT_NAME := admin_integration
build: ## Build SeaweedFS with latest changes and create Docker image
@echo "🔨 Building SeaweedFS with latest changes..."
@echo "1️⃣ Generating admin templates..."
@cd ../../ && make admin-generate
@echo "2️⃣ Building Docker image with latest changes..."
@cd ../ && make build
@echo "3️⃣ Copying binary for local docker-compose..."
@cp ../weed ./weed-local
@echo "✅ Build complete! Updated image: chrislusf/seaweedfs:local"
@echo "💡 Run 'make restart' to apply changes to running services"
build-and-restart: build ## Build with latest changes and restart services
@echo "🔄 Recreating services with new image..."
@echo "1️⃣ Recreating admin server with new image..."
@docker-compose -f $(COMPOSE_FILE) up -d admin
@sleep 5
@echo "2️⃣ Recreating workers to reconnect..."
@docker-compose -f $(COMPOSE_FILE) up -d worker1 worker2 worker3
@echo "✅ All services recreated with latest changes!"
@echo "🌐 Admin UI: http://localhost:23646/"
@echo "💡 Workers will reconnect to the new admin server"
restart-workers: ## Restart all workers to reconnect to admin server
@echo "🔄 Restarting workers to reconnect to admin server..."
@docker-compose -f $(COMPOSE_FILE) restart worker1 worker2 worker3
@echo "✅ Workers restarted and will reconnect to admin server"
help: ## Show this help message
@echo "SeaweedFS Admin Integration Test"
@echo "================================"
@echo "Tests admin server task distribution to workers using official weed commands"
@echo ""
@echo "🏗️ Cluster Management:"
@grep -E '^(start|stop|restart|clean|status|build):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "🧪 Testing:"
@grep -E '^(test|demo|validate|quick-test):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "🗑️ Vacuum Testing:"
@grep -E '^vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "📜 Monitoring:"
@grep -E '^(logs|admin-logs|worker-logs|master-logs|admin-ui):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}'
@echo ""
@echo "🚀 Quick Start:"
@echo " make start # Start cluster"
@echo " make vacuum-test # Test vacuum tasks"
@echo " make vacuum-help # Vacuum testing guide"
@echo ""
@echo "💡 For detailed vacuum testing: make vacuum-help"
start: ## Start the complete SeaweedFS cluster with admin and workers
@echo "🚀 Starting SeaweedFS cluster with admin and workers..."
@docker-compose -f $(COMPOSE_FILE) up -d
@echo "✅ Cluster started!"
@echo ""
@echo "📊 Access points:"
@echo " • Admin UI: http://localhost:23646/"
@echo " • Master UI: http://localhost:9333/"
@echo " • Filer: http://localhost:8888/"
@echo ""
@echo "📈 Services starting up..."
@echo " • Master server: ✓"
@echo " • Volume servers: Starting (6 servers)..."
@echo " • Filer: Starting..."
@echo " • Admin server: Starting..."
@echo " • Workers: Starting (3 workers)..."
@echo ""
@echo "⏳ Use 'make status' to check startup progress"
@echo "💡 Use 'make logs' to watch the startup process"
start-staged: ## Start services in proper order with delays
@echo "🚀 Starting SeaweedFS cluster in stages..."
@echo ""
@echo "Stage 1: Starting Master server..."
@docker-compose -f $(COMPOSE_FILE) up -d master
@sleep 10
@echo ""
@echo "Stage 2: Starting Volume servers..."
@docker-compose -f $(COMPOSE_FILE) up -d volume1 volume2 volume3 volume4 volume5 volume6
@sleep 15
@echo ""
@echo "Stage 3: Starting Filer..."
@docker-compose -f $(COMPOSE_FILE) up -d filer
@sleep 10
@echo ""
@echo "Stage 4: Starting Admin server..."
@docker-compose -f $(COMPOSE_FILE) up -d admin
@sleep 15
@echo ""
@echo "Stage 5: Starting Workers..."
@docker-compose -f $(COMPOSE_FILE) up -d worker1 worker2 worker3
@sleep 10
@echo ""
@echo "Stage 6: Starting Load generator and Monitor..."
@docker-compose -f $(COMPOSE_FILE) up -d load_generator monitor
@echo ""
@echo "✅ All services started!"
@echo ""
@echo "📊 Access points:"
@echo " • Admin UI: http://localhost:23646/"
@echo " • Master UI: http://localhost:9333/"
@echo " • Filer: http://localhost:8888/"
@echo ""
@echo "⏳ Services are initializing... Use 'make status' to check progress"
stop: ## Stop all services
@echo "🛑 Stopping SeaweedFS cluster..."
@docker-compose -f $(COMPOSE_FILE) down
@echo "✅ Cluster stopped"
restart: stop start ## Restart the entire cluster
clean: ## Stop and remove all containers, networks, and volumes
@echo "🧹 Cleaning up SeaweedFS test environment..."
@docker-compose -f $(COMPOSE_FILE) down -v --remove-orphans
@docker system prune -f
@rm -rf data/
@echo "✅ Environment cleaned"
status: ## Check the status of all services
@echo "📊 SeaweedFS Cluster Status"
@echo "=========================="
@docker-compose -f $(COMPOSE_FILE) ps
@echo ""
@echo "📋 Service Health:"
@echo "Master:"
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' 2>/dev/null || echo " ❌ Master not ready"
@echo "Admin:"
@curl -s http://localhost:23646/ | grep -q "Admin" && echo " ✅ Admin ready" || echo " ❌ Admin not ready"
logs: ## Show logs from all services
@echo "📜 Following logs from all services..."
@echo "💡 Press Ctrl+C to stop following logs"
@docker-compose -f $(COMPOSE_FILE) logs -f
admin-logs: ## Show logs from admin server only
@echo "📜 Admin server logs:"
@docker-compose -f $(COMPOSE_FILE) logs -f admin
worker-logs: ## Show logs from all workers
@echo "📜 Worker logs:"
@docker-compose -f $(COMPOSE_FILE) logs -f worker1 worker2 worker3
master-logs: ## Show logs from master server
@echo "📜 Master server logs:"
@docker-compose -f $(COMPOSE_FILE) logs -f master
admin-ui: ## Open admin UI in browser (macOS)
@echo "🌐 Opening admin UI in browser..."
@open http://localhost:23646/ || echo "💡 Manually open: http://localhost:23646/"
test: ## Run integration test to verify task assignment and completion
@echo "🧪 Running Admin-Worker Integration Test"
@echo "========================================"
@echo ""
@echo "1️⃣ Checking cluster health..."
@sleep 5
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master healthy" || echo "❌ Master not ready"
@curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin healthy" || echo "❌ Admin not ready"
@echo ""
@echo "2️⃣ Checking worker registration..."
@sleep 10
@echo "💡 Check admin UI for connected workers: http://localhost:23646/"
@echo ""
@echo "3️⃣ Generating load to trigger EC tasks..."
@echo "📝 Creating test files to fill volumes..."
@echo "Creating large files with random data to trigger EC (targeting ~60MB total to exceed 50MB limit)..."
@for i in {1..12}; do \
echo "Creating 5MB random file $$i..."; \
docker run --rm --network admin_integration_seaweed_net -v /tmp:/tmp --entrypoint sh chrislusf/seaweedfs:local -c "dd if=/dev/urandom of=/tmp/largefile$$i.dat bs=1M count=5 2>/dev/null && weed upload -master=master:9333 /tmp/largefile$$i.dat && rm /tmp/largefile$$i.dat"; \
sleep 3; \
done
@echo ""
@echo "4️⃣ Waiting for volumes to process large files and reach 50MB limit..."
@echo "This may take a few minutes as we're uploading 60MB of data..."
@sleep 60
@echo ""
@echo "5️⃣ Checking for EC task creation and assignment..."
@echo "💡 Monitor the admin UI to see:"
@echo " • Tasks being created for volumes needing EC"
@echo " • Workers picking up tasks"
@echo " • Task progress (pending → running → completed)"
@echo " • EC shards being distributed"
@echo ""
@echo "✅ Integration test setup complete!"
@echo "📊 Monitor progress at: http://localhost:23646/"
quick-test: ## Quick verification that core services are running
@echo "⚡ Quick Health Check"
@echo "===================="
@echo "Master: $$(curl -s http://localhost:9333/cluster/status | jq -r '.IsLeader // "not ready"')"
@echo "Admin: $$(curl -s http://localhost:23646/ | grep -q "Admin" && echo "ready" || echo "not ready")"
@echo "Workers: $$(docker-compose -f $(COMPOSE_FILE) ps worker1 worker2 worker3 | grep -c Up) running"
validate: ## Validate integration test configuration
@echo "🔍 Validating Integration Test Configuration"
@echo "==========================================="
@chmod +x test-integration.sh
@./test-integration.sh
demo: start ## Start cluster and run demonstration
@echo "🎭 SeaweedFS Admin-Worker Demo"
@echo "============================="
@echo ""
@echo "⏳ Waiting for services to start..."
@sleep 45
@echo ""
@echo "🎯 Demo Overview:"
@echo " • 1 Master server (coordinates cluster)"
@echo " • 6 Volume servers (50MB volume limit)"
@echo " • 1 Admin server (task management)"
@echo " • 3 Workers (execute EC tasks)"
@echo " • Load generator (creates files continuously)"
@echo ""
@echo "📊 Watch the process:"
@echo " 1. Visit: http://localhost:23646/"
@echo " 2. Observe workers connecting"
@echo " 3. Watch tasks being created and assigned"
@echo " 4. See tasks progress from pending → completed"
@echo ""
@echo "🔄 The demo will:"
@echo " • Fill volumes to 50MB limit"
@echo " • Admin detects volumes needing EC"
@echo " • Workers receive and execute EC tasks"
@echo " • Tasks complete with shard distribution"
@echo ""
@echo "💡 Use 'make worker-logs' to see worker activity"
@echo "💡 Use 'make admin-logs' to see admin task management"
# Vacuum Testing Targets
vacuum-test: ## Create test data with garbage and verify vacuum detection
@echo "🧪 SeaweedFS Vacuum Task Testing"
@echo "================================"
@echo ""
@echo "1️⃣ Checking cluster health..."
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master ready" || (echo "❌ Master not ready. Run 'make start' first." && exit 1)
@curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin ready" || (echo "❌ Admin not ready. Run 'make start' first." && exit 1)
@echo ""
@echo "2️⃣ Creating test data with garbage..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=25 -delete=0.5 -size=200
@echo ""
@echo "3️⃣ Configuration Instructions:"
@echo " Visit: http://localhost:23646/maintenance/config/vacuum"
@echo " Set for testing:"
@echo " • Enable Vacuum Tasks: ✅ Checked"
@echo " • Garbage Threshold: 0.20 (20%)"
@echo " • Scan Interval: [30] [Seconds]"
@echo " • Min Volume Age: [0] [Minutes]"
@echo " • Max Concurrent: 2"
@echo ""
@echo "4️⃣ Monitor vacuum tasks at: http://localhost:23646/maintenance"
@echo ""
@echo "💡 Use 'make vacuum-status' to check volume garbage ratios"
vacuum-demo: ## Run automated vacuum testing demonstration
@echo "🎭 Vacuum Task Demo"
@echo "=================="
@echo ""
@echo "⚠️ This demo requires user interaction for configuration"
@echo "💡 Make sure cluster is running with 'make start'"
@echo ""
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester sh -c "chmod +x demo_vacuum_testing.sh && ./demo_vacuum_testing.sh"
vacuum-status: ## Check current volume status and garbage ratios
@echo "📊 Current Volume Status"
@echo "======================="
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester sh -c "chmod +x check_volumes.sh && ./check_volumes.sh"
vacuum-data: ## Create test data with configurable parameters
@echo "📁 Creating vacuum test data..."
@echo "Usage: make vacuum-data [FILES=20] [DELETE=0.4] [SIZE=100]"
@echo ""
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go \
-files=$${FILES:-20} \
-delete=$${DELETE:-0.4} \
-size=$${SIZE:-100}
vacuum-data-high: ## Create high garbage ratio test data (should trigger vacuum)
@echo "📁 Creating high garbage test data (70% garbage)..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=30 -delete=0.7 -size=150
vacuum-data-low: ## Create low garbage ratio test data (should NOT trigger vacuum)
@echo "📁 Creating low garbage test data (15% garbage)..."
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=30 -delete=0.15 -size=150
vacuum-continuous: ## Generate garbage continuously for testing
@echo "🔄 Generating continuous garbage for vacuum testing..."
@echo "Creating 5 rounds of test data with 30-second intervals..."
@for i in {1..5}; do \
echo "Round $$i: Creating garbage..."; \
docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=10 -delete=0.6 -size=100; \
echo "Waiting 30 seconds..."; \
sleep 30; \
done
@echo "✅ Continuous test complete. Check vacuum task activity!"
vacuum-clean: ## Clean up vacuum test data (removes all volumes!)
@echo "🧹 Cleaning up vacuum test data..."
@echo "⚠️ WARNING: This will delete ALL volumes!"
@read -p "Are you sure? (y/N): " confirm && [ "$$confirm" = "y" ] || exit 1
@echo "Stopping cluster..."
@docker-compose -f $(COMPOSE_FILE) down
@echo "Removing volume data..."
@rm -rf data/volume*/
@echo "Restarting cluster..."
@docker-compose -f $(COMPOSE_FILE) up -d
@echo "✅ Clean up complete. Fresh volumes ready for testing."
vacuum-help: ## Show vacuum testing help and examples
@echo "🧪 Vacuum Testing Commands (Docker-based)"
@echo "=========================================="
@echo ""
@echo "Quick Start:"
@echo " make start # Start SeaweedFS cluster with vacuum-tester"
@echo " make vacuum-test # Create test data and instructions"
@echo " make vacuum-status # Check volume status"
@echo ""
@echo "Data Generation:"
@echo " make vacuum-data-high # High garbage (should trigger)"
@echo " make vacuum-data-low # Low garbage (should NOT trigger)"
@echo " make vacuum-continuous # Continuous garbage generation"
@echo ""
@echo "Monitoring:"
@echo " make vacuum-status # Quick volume status check"
@echo " make vacuum-demo # Full guided demonstration"
@echo ""
@echo "Configuration:"
@echo " Visit: http://localhost:23646/maintenance/config/vacuum"
@echo " Monitor: http://localhost:23646/maintenance"
@echo ""
@echo "Custom Parameters:"
@echo " make vacuum-data FILES=50 DELETE=0.8 SIZE=200"
@echo ""
@echo "💡 All commands now run inside Docker containers"
@echo "Documentation:"
@echo " See: VACUUM_TEST_README.md for complete guide"
-32
View File
@@ -1,32 +0,0 @@
#!/bin/sh
echo "📊 Quick Volume Status Check"
echo "============================"
echo ""
# Check if master is running
MASTER_URL="${MASTER_HOST:-master:9333}"
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then
echo "❌ Master server not available at $MASTER_URL"
exit 1
fi
echo "🔍 Fetching volume status from master..."
curl -s "http://$MASTER_URL/vol/status" | jq -r '
if .Volumes and .Volumes.DataCenters then
.Volumes.DataCenters | to_entries[] | .value | to_entries[] | .value | to_entries[] | .value | if . then .[] else empty end |
"Volume \(.Id):
Size: \(.Size | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end)
Files: \(.FileCount) active, \(.DeleteCount) deleted
Garbage: \(.DeletedByteCount | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end) (\(if .Size > 0 then (.DeletedByteCount / .Size * 100 | floor) else 0 end)%)
Status: \(if (.DeletedByteCount / .Size * 100) > 30 then "🎯 NEEDS VACUUM" else "✅ OK" end)
"
else
"No volumes found"
end'
echo ""
echo "💡 Legend:"
echo " 🎯 NEEDS VACUUM: >30% garbage ratio"
echo " ✅ OK: <30% garbage ratio"
echo ""
@@ -1,280 +0,0 @@
package main
import (
"bytes"
"crypto/rand"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"time"
)
var (
master = flag.String("master", "master:9333", "SeaweedFS master server address")
fileCount = flag.Int("files", 20, "Number of files to create")
deleteRatio = flag.Float64("delete", 0.4, "Ratio of files to delete (0.0-1.0)")
fileSizeKB = flag.Int("size", 100, "Size of each file in KB")
)
type AssignResult struct {
Fid string `json:"fid"`
Url string `json:"url"`
PublicUrl string `json:"publicUrl"`
Count int `json:"count"`
Error string `json:"error"`
}
func main() {
flag.Parse()
fmt.Println("🧪 Creating fake data for vacuum task testing...")
fmt.Printf("Master: %s\n", *master)
fmt.Printf("Files to create: %d\n", *fileCount)
fmt.Printf("Delete ratio: %.1f%%\n", *deleteRatio*100)
fmt.Printf("File size: %d KB\n", *fileSizeKB)
fmt.Println()
if *fileCount == 0 {
// Just check volume status
fmt.Println("📊 Checking volume status...")
checkVolumeStatus()
return
}
// Step 1: Create test files
fmt.Println("📁 Step 1: Creating test files...")
fids := createTestFiles()
// Step 2: Delete some files to create garbage
fmt.Println("🗑️ Step 2: Deleting files to create garbage...")
deleteFiles(fids)
// Step 3: Check volume status
fmt.Println("📊 Step 3: Checking volume status...")
checkVolumeStatus()
// Step 4: Configure vacuum for testing
fmt.Println("⚙️ Step 4: Instructions for testing...")
printTestingInstructions()
}
func createTestFiles() []string {
var fids []string
for i := 0; i < *fileCount; i++ {
// Generate random file content
fileData := make([]byte, *fileSizeKB*1024)
rand.Read(fileData)
// Get file ID assignment
assign, err := assignFileId()
if err != nil {
log.Printf("Failed to assign file ID for file %d: %v", i, err)
continue
}
// Upload file
err = uploadFile(assign, fileData, fmt.Sprintf("test_file_%d.dat", i))
if err != nil {
log.Printf("Failed to upload file %d: %v", i, err)
continue
}
fids = append(fids, assign.Fid)
if (i+1)%5 == 0 {
fmt.Printf(" Created %d/%d files...\n", i+1, *fileCount)
}
}
fmt.Printf("✅ Created %d files successfully\n\n", len(fids))
return fids
}
func deleteFiles(fids []string) {
deleteCount := int(float64(len(fids)) * *deleteRatio)
for i := 0; i < deleteCount; i++ {
err := deleteFile(fids[i])
if err != nil {
log.Printf("Failed to delete file %s: %v", fids[i], err)
continue
}
if (i+1)%5 == 0 {
fmt.Printf(" Deleted %d/%d files...\n", i+1, deleteCount)
}
}
fmt.Printf("✅ Deleted %d files (%.1f%% of total)\n\n", deleteCount, *deleteRatio*100)
}
func assignFileId() (*AssignResult, error) {
resp, err := http.Get(fmt.Sprintf("http://%s/dir/assign", *master))
if err != nil {
return nil, err
}
defer resp.Body.Close()
var result AssignResult
err = json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return nil, err
}
if result.Error != "" {
return nil, fmt.Errorf("assignment error: %s", result.Error)
}
return &result, nil
}
func uploadFile(assign *AssignResult, data []byte, filename string) error {
url := fmt.Sprintf("http://%s/%s", assign.Url, assign.Fid)
body := &bytes.Buffer{}
body.Write(data)
req, err := http.NewRequest("POST", url, body)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
if filename != "" {
req.Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body))
}
return nil
}
func deleteFile(fid string) error {
url := fmt.Sprintf("http://%s/%s", *master, fid)
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return err
}
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func checkVolumeStatus() {
// Get volume list from master
resp, err := http.Get(fmt.Sprintf("http://%s/vol/status", *master))
if err != nil {
log.Printf("Failed to get volume status: %v", err)
return
}
defer resp.Body.Close()
var volumes map[string]interface{}
err = json.NewDecoder(resp.Body).Decode(&volumes)
if err != nil {
log.Printf("Failed to decode volume status: %v", err)
return
}
fmt.Println("📊 Volume Status Summary:")
if vols, ok := volumes["Volumes"].([]interface{}); ok {
for _, vol := range vols {
if v, ok := vol.(map[string]interface{}); ok {
id := int(v["Id"].(float64))
size := uint64(v["Size"].(float64))
fileCount := int(v["FileCount"].(float64))
deleteCount := int(v["DeleteCount"].(float64))
deletedBytes := uint64(v["DeletedByteCount"].(float64))
garbageRatio := 0.0
if size > 0 {
garbageRatio = float64(deletedBytes) / float64(size) * 100
}
fmt.Printf(" Volume %d:\n", id)
fmt.Printf(" Size: %s\n", formatBytes(size))
fmt.Printf(" Files: %d (active), %d (deleted)\n", fileCount, deleteCount)
fmt.Printf(" Garbage: %s (%.1f%%)\n", formatBytes(deletedBytes), garbageRatio)
if garbageRatio > 30 {
fmt.Printf(" 🎯 This volume should trigger vacuum (>30%% garbage)\n")
}
fmt.Println()
}
}
}
}
func formatBytes(bytes uint64) string {
if bytes < 1024 {
return fmt.Sprintf("%d B", bytes)
} else if bytes < 1024*1024 {
return fmt.Sprintf("%.1f KB", float64(bytes)/1024)
} else if bytes < 1024*1024*1024 {
return fmt.Sprintf("%.1f MB", float64(bytes)/(1024*1024))
} else {
return fmt.Sprintf("%.1f GB", float64(bytes)/(1024*1024*1024))
}
}
func printTestingInstructions() {
fmt.Println("🧪 Testing Instructions:")
fmt.Println()
fmt.Println("1. Configure Vacuum for Testing:")
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum")
fmt.Println(" Set:")
fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n")
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n")
fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n")
fmt.Printf(" - Max Concurrent: 2\n")
fmt.Printf(" - Min Interval: 1m (faster repeat)\n")
fmt.Println()
fmt.Println("2. Monitor Vacuum Tasks:")
fmt.Println(" Visit: http://localhost:23646/maintenance")
fmt.Println(" Watch for vacuum tasks to appear in the queue")
fmt.Println()
fmt.Println("3. Manual Vacuum (Optional):")
fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.20'")
fmt.Println(" (Note: Master API still uses 0.0-1.0 decimal format)")
fmt.Println()
fmt.Println("4. Check Logs:")
fmt.Println(" Look for messages like:")
fmt.Println(" - 'Vacuum detector found X volumes needing vacuum'")
fmt.Println(" - 'Applied vacuum configuration'")
fmt.Println(" - 'Worker executing task: vacuum'")
fmt.Println()
fmt.Println("5. Verify Results:")
fmt.Println(" Re-run this script with -files=0 to check volume status")
fmt.Println(" Garbage ratios should decrease after vacuum operations")
fmt.Println()
fmt.Printf("🚀 Quick test command:\n")
fmt.Printf(" go run create_vacuum_test_data.go -files=0\n")
fmt.Println()
}
@@ -1,105 +0,0 @@
#!/bin/sh
echo "🧪 SeaweedFS Vacuum Task Testing Demo"
echo "======================================"
echo ""
# Check if SeaweedFS is running
echo "📋 Checking SeaweedFS status..."
MASTER_URL="${MASTER_HOST:-master:9333}"
ADMIN_URL="${ADMIN_HOST:-admin:23646}"
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then
echo "❌ SeaweedFS master not running at $MASTER_URL"
echo " Please ensure Docker cluster is running: make start"
exit 1
fi
if ! curl -s http://volume1:8080/status > /dev/null; then
echo "❌ SeaweedFS volume servers not running"
echo " Please ensure Docker cluster is running: make start"
exit 1
fi
if ! curl -s http://$ADMIN_URL/ > /dev/null; then
echo "❌ SeaweedFS admin server not running at $ADMIN_URL"
echo " Please ensure Docker cluster is running: make start"
exit 1
fi
echo "✅ All SeaweedFS components are running"
echo ""
# Phase 1: Create test data
echo "📁 Phase 1: Creating test data with garbage..."
go run create_vacuum_test_data.go -master=$MASTER_URL -files=15 -delete=0.5 -size=150
echo ""
# Phase 2: Check initial status
echo "📊 Phase 2: Checking initial volume status..."
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0
echo ""
# Phase 3: Configure vacuum
echo "⚙️ Phase 3: Vacuum configuration instructions..."
echo " 1. Visit: http://localhost:23646/maintenance/config/vacuum"
echo " 2. Set these values for testing:"
echo " - Enable Vacuum Tasks: ✅ Checked"
echo " - Garbage Threshold: 0.30"
echo " - Scan Interval: [30] [Seconds]"
echo " - Min Volume Age: [0] [Minutes]"
echo " - Max Concurrent: 2"
echo " 3. Click 'Save Configuration'"
echo ""
read -p " Press ENTER after configuring vacuum settings..."
echo ""
# Phase 4: Monitor tasks
echo "🎯 Phase 4: Monitoring vacuum tasks..."
echo " Visit: http://localhost:23646/maintenance"
echo " You should see vacuum tasks appear within 30 seconds"
echo ""
echo " Waiting 60 seconds for vacuum detection and execution..."
for i in {60..1}; do
printf "\r Countdown: %02d seconds" $i
sleep 1
done
echo ""
echo ""
# Phase 5: Check results
echo "📈 Phase 5: Checking results after vacuum..."
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0
echo ""
# Phase 6: Create more garbage for continuous testing
echo "🔄 Phase 6: Creating additional garbage for continuous testing..."
echo " Running 3 rounds of garbage creation..."
for round in {1..3}; do
echo " Round $round: Creating garbage..."
go run create_vacuum_test_data.go -master=$MASTER_URL -files=8 -delete=0.6 -size=100
echo " Waiting 30 seconds before next round..."
sleep 30
done
echo ""
echo "📊 Final volume status:"
go run create_vacuum_test_data.go -master=$MASTER_URL -files=0
echo ""
echo "🎉 Demo Complete!"
echo ""
echo "🔍 Things to check:"
echo " 1. Maintenance Queue: http://localhost:23646/maintenance"
echo " 2. Volume Status: http://localhost:9333/vol/status"
echo " 3. Admin Dashboard: http://localhost:23646"
echo ""
echo "💡 Next Steps:"
echo " - Try different garbage thresholds (0.10, 0.50, 0.80)"
echo " - Adjust scan intervals (10s, 1m, 5m)"
echo " - Monitor logs for vacuum operations"
echo " - Test with multiple volumes"
echo ""
@@ -1,240 +0,0 @@
name: admin_integration
networks:
seaweed_net:
driver: bridge
services:
master:
image: chrislusf/seaweedfs:local
ports:
- "9333:9333"
- "19333:19333"
command: "master -ip=master -mdir=/data -volumeSizeLimitMB=50"
environment:
- WEED_MASTER_VOLUME_GROWTH_COPY_1=1
- WEED_MASTER_VOLUME_GROWTH_COPY_2=2
- WEED_MASTER_VOLUME_GROWTH_COPY_OTHER=1
volumes:
- ./data/master:/data
networks:
- seaweed_net
volume1:
image: chrislusf/seaweedfs:local
ports:
- "8080:8080"
- "18080:18080"
command: "volume -master=master:9333 -ip=volume1 -dir=/data -max=10"
depends_on:
- master
volumes:
- ./data/volume1:/data
networks:
- seaweed_net
volume2:
image: chrislusf/seaweedfs:local
ports:
- "8081:8080"
- "18081:18080"
command: "volume -master=master:9333 -ip=volume2 -dir=/data -max=10"
depends_on:
- master
volumes:
- ./data/volume2:/data
networks:
- seaweed_net
volume3:
image: chrislusf/seaweedfs:local
ports:
- "8082:8080"
- "18082:18080"
command: "volume -master=master:9333 -ip=volume3 -dir=/data -max=10"
depends_on:
- master
volumes:
- ./data/volume3:/data
networks:
- seaweed_net
volume4:
image: chrislusf/seaweedfs:local
ports:
- "8083:8080"
- "18083:18080"
command: "volume -master=master:9333 -ip=volume4 -dir=/data -max=10"
depends_on:
- master
volumes:
- ./data/volume4:/data
networks:
- seaweed_net
volume5:
image: chrislusf/seaweedfs:local
ports:
- "8084:8080"
- "18084:18080"
command: "volume -master=master:9333 -ip=volume5 -dir=/data -max=10"
depends_on:
- master
volumes:
- ./data/volume5:/data
networks:
- seaweed_net
volume6:
image: chrislusf/seaweedfs:local
ports:
- "8085:8080"
- "18085:18080"
command: "volume -master=master:9333 -ip=volume6 -dir=/data -max=10"
depends_on:
- master
volumes:
- ./data/volume6:/data
networks:
- seaweed_net
filer:
image: chrislusf/seaweedfs:local
ports:
- "8888:8888"
- "18888:18888"
command: "filer -master=master:9333 -ip=filer"
depends_on:
- master
volumes:
- ./data/filer:/data
networks:
- seaweed_net
admin:
image: chrislusf/seaweedfs:local
ports:
- "23646:23646" # HTTP admin interface (default port)
- "33646:33646" # gRPC worker communication (23646 + 10000)
command: "-v=2 admin -port=23646 -masters=master:9333 -dataDir=/data"
depends_on:
- master
- filer
volumes:
- ./data/admin:/data
networks:
- seaweed_net
worker1:
image: chrislusf/seaweedfs:local
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
depends_on:
- admin
volumes:
- ./data/worker1:/data
networks:
- seaweed_net
environment:
- WORKER_ID=worker-1
worker2:
image: chrislusf/seaweedfs:local
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
depends_on:
- admin
volumes:
- ./data/worker2:/data
networks:
- seaweed_net
environment:
- WORKER_ID=worker-2
worker3:
image: chrislusf/seaweedfs:local
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
depends_on:
- admin
volumes:
- ./data/worker3:/data
networks:
- seaweed_net
environment:
- WORKER_ID=worker-3
load_generator:
image: chrislusf/seaweedfs:local
entrypoint: ["/bin/sh"]
command: >
-c "
echo 'Starting load generator...';
sleep 30;
echo 'Generating continuous load with 50MB volume limit...';
while true; do
echo 'Writing test files...';
echo 'Test file content at $(date)' | /usr/bin/weed upload -server=master:9333;
sleep 5;
echo 'Deleting some files...';
/usr/bin/weed shell -master=master:9333 <<< 'fs.rm /test_file_*' || true;
sleep 10;
done
"
depends_on:
- master
- filer
- admin
networks:
- seaweed_net
monitor:
image: alpine:latest
entrypoint: ["/bin/sh"]
command: >
-c "
apk add --no-cache curl jq;
echo 'Starting cluster monitor...';
sleep 30;
while true; do
echo '=== Cluster Status $(date) ===';
echo 'Master status:';
curl -s http://master:9333/cluster/status | jq '.IsLeader, .Peers' || echo 'Master not ready';
echo;
echo 'Admin status:';
curl -s http://admin:23646/ | grep -o 'Admin.*Interface' || echo 'Admin not ready';
echo;
echo 'Volume count by server:';
curl -s http://master:9333/vol/status | jq '.Volumes | length' || echo 'Volumes not ready';
echo;
sleep 60;
done
"
depends_on:
- master
- admin
- filer
networks:
- seaweed_net
vacuum-tester:
image: chrislusf/seaweedfs:local
entrypoint: ["/bin/sh"]
command: >
-c "
echo 'Installing dependencies for vacuum testing...';
apk add --no-cache jq curl go bash;
echo 'Vacuum tester ready...';
echo 'Use: docker-compose exec vacuum-tester sh';
echo 'Available commands: go, weed, curl, jq, bash, sh';
sleep infinity
"
depends_on:
- master
- admin
- filer
volumes:
- .:/testing
working_dir: /testing
networks:
- seaweed_net
environment:
- MASTER_HOST=master:9333
- ADMIN_HOST=admin:23646

Some files were not shown because too many files have changed in this diff Show More