Compare commits

..
Author SHA1 Message Date
Chris LuandCopilot 8500a3bc56 ci: remove s3tests skip - test will be fixed in s3-tests repo
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 03:23:36 -08:00
Chris LuandCopilot 678c52e74f ci: skip s3tests test expecting incorrect delete behavior
Skip test_object_lock_delete_object_with_retention_and_marker in s3tests because it expects SeaweedFS incorrect behavior (allowing delete under COMPLIANCE retention). SeaweedFS now correctly implements AWS S3 behavior: delete of objects under COMPLIANCE retention returns AccessDenied and does not create a delete marker.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 03:20:05 -08:00
Chris LuandCopilot 8fac0cdab8 test: fix retention tests to expect correct AWS S3 behavior
Tests were expecting SeaweedFS old (incorrect) behavior where DeleteObject succeeded under COMPLIANCE retention. Updated to expect correct AWS S3 behavior: simple DELETE (without versionId) is blocked by active COMPLIANCE/legal-hold, returning AccessDenied instead of creating a delete marker.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:57:31 -08:00
Chris LuandCopilot 09bfb28db9 s3api: simplify objectLockVersionToCheckForDelete to single return value
Since all branches now return true (fail-closed), eliminate the boolean return value and the dead-code if guards. The function now clearly indicates that object lock checks always execute, with no opt-out path.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:08:32 -08:00
Chris LuandCopilot 3bd990d2ea s3api: address object lock delete review comments
- Remove versioningConfigured guard: object lock protections must apply to all buckets
- Combine identical switch cases for clarity
- Change default case to fail-closed (true) for safety
- Add test case for suspended versioning with specific versionId
- Update test expectations to reflect fail-closed default

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:00:30 -08:00
Chris LuandCopilot 5e040ba03d s3api: enforce object lock protections on delete (COMPLIANCE/GOVERNANCE)
Enforce object lock protections before creating delete markers; active COMPLIANCE retention denies deletes and active GOVERNANCE retention denies deletes unless bypass is authorized. Includes unit tests.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 01:52:08 -08:00
2636 changed files with 90983 additions and 438735 deletions
-22
View File
@@ -1,22 +0,0 @@
[codespell]
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
skip = .git,.git-meta,.gitignore,.gitattributes,*.svg,go.sum,vendor,*.lock,*.css,*.min.*,.codespellrc,*_templ.go
check-hidden = true
# Ignore camelCase and PascalCase identifiers (very common in Go/Rust/JS
# source, e.g. allLocations, publishErr, ReadInside, FlushInterval).
ignore-regex = \b[a-z]+[A-Z]\w*\b|\b[A-Z][a-z]+[A-Z]\w*\b
# visibles: variable name for VisibleInterval collections in filer/mount code
# fo: `*FilerOptions` receiver name (e.g. `func (fo *FilerOptions) ...`)
# te: "truncate error" local variable (e.g. `if te := w.Truncate(end); te != nil`)
# ser: Rust serde serializer variable (`serde_json::ser`, `let mut ser = ...`)
# bject: intentional wildcard test data (e.g. `s3:Get?bject` matching `s3:GetObject`)
# unparseable: accepted alternate spelling used throughout the codebase
# keep-alives: correct plural of the technical term (SSH/HTTP keep-alive)
# tread: valid English word in the idiom "tread carefully" (help text)
# anc: variable abbreviation for "ancestor" in tree/path tests
# ue: appears inside JSON test fixtures with embedded escaped quotes (Bl\"ue)
# auther: local variable meaning "authenticator" (tls.go: `auther := Authenticator{}`)
# thirdparty: literal Maven groupId `org.apache.hadoop.thirdparty` (external, cannot rename)
# unknwon: GitHub username / Go module path (`github.com/unknwon/goconfig`)
# atleast: CLI mode literal string in test/benchmark/fuse_db/bin/sqlite_verify.py
ignore-words-list = visibles,fo,te,ser,bject,unparseable,keep-alives,tread,anc,ue,auther,thirdparty,unknwon,atleast
@@ -1,66 +0,0 @@
name: Fix fusermount3 setuid
description: >
Make sure the fusermount3 an unprivileged mount will find can actually mount.
Some runner images carry a second, source-built fusermount3 in /usr/local/bin
that shadows the distro one in PATH; it is neither setuid nor root-owned, so
every mount fails with "mount failed: Operation not permitted". Both the Go
mount and sw-fuse resolve the helper through PATH.
Run it after the step that apt-installs fuse3.
runs:
using: composite
steps:
- name: Point PATH at a fusermount3 that can mount
shell: bash
run: |
set -euo pipefail
# setuid only grants root when root owns the file, and exec follows
# symlinks, so judge the target.
can_mount() {
[ -u "$1" ] && [ "$(stat -Lc %u "$1")" = 0 ]
}
bin=$(command -v fusermount3 || true)
if [ -z "$bin" ]; then
echo "no fusermount3 in PATH" >&2
exit 1
fi
if can_mount "$bin"; then
ls -l "$bin"
exit 0
fi
echo "$bin cannot mount unprivileged:"
ls -l "$bin"
# The distro fusermount3 is setuid root and is the one meant to be used.
# Reach it through a symlink earlier in PATH: exec resolves the link, so
# the target keeps its setuid bit, and nothing on the image is modified.
for distro in /usr/bin/fusermount3 /bin/fusermount3; do
if [ "$distro" != "$bin" ] && can_mount "$distro"; then
mkdir -p "$RUNNER_TEMP/fuse-bin"
ln -sf "$distro" "$RUNNER_TEMP/fuse-bin/fusermount3"
echo "$RUNNER_TEMP/fuse-bin" >> "$GITHUB_PATH"
echo "using $distro instead"
ls -l "$distro"
exit 0
fi
done
# No usable distro binary, so setting the bit is the only way out - but
# the target came from PATH: do that only for a root-owned system binary,
# never for anything else that happens to sit there.
case "$bin" in
/bin/*|/sbin/*|/usr/bin/*|/usr/sbin/*|/usr/local/bin/*|/usr/local/sbin/*) ;;
*) echo "refusing to setuid $bin: outside the system bin paths" >&2; exit 1 ;;
esac
if [ -L "$bin" ] || [ ! -f "$bin" ] || [ ! -x "$bin" ]; then
echo "refusing to setuid $bin: not a regular executable file" >&2
exit 1
fi
if [ "$(stat -c %u "$bin")" != 0 ] || [ "$(stat -c %g "$bin")" != 0 ]; then
echo "refusing to setuid $bin: not owned by root:root" >&2
exit 1
fi
sudo chmod u+s "$bin"
ls -l "$bin"
+2 -12
View File
@@ -3,15 +3,6 @@ name: "go: build dev binaries"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/binaries_dev.yml'
concurrency:
group: binaries-dev-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
@@ -27,7 +18,6 @@ jobs:
- name: Delete old release assets
uses: mknejp/delete-release-assets@v1
continue-on-error: true
with:
token: ${{ github.token }}
tag: dev
@@ -48,7 +38,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
@@ -97,7 +87,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
with:
+4 -9
View File
@@ -2,11 +2,6 @@ name: "Code Scanning - Action"
on:
pull_request:
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- '.github/workflows/codeql.yml'
concurrency:
group: ${{ github.head_ref }}/codeql
@@ -23,11 +18,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4.37.4
uses: github/codeql-action/init@v4
# Override language selection by uncommenting this and choosing your languages
with:
languages: go
@@ -35,7 +30,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below).
- name: Autobuild
uses: github/codeql-action/autobuild@v4.37.4
uses: github/codeql-action/autobuild@v4
# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -49,4 +44,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4.37.4
uses: github/codeql-action/analyze@v4
-23
View File
@@ -1,23 +0,0 @@
# Codespell configuration is within .codespellrc
---
name: Codespell
on:
push:
branches: [master]
pull_request:
branches: [master]
permissions:
contents: read
jobs:
codespell:
name: Check for spelling errors
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Codespell
uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2
+23 -109
View File
@@ -3,109 +3,23 @@ name: "docker: build dev containers"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'seaweed-volume/**'
- 'docker/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/container_dev.yml'
permissions:
contents: read
jobs:
# ── Pre-build Rust volume server binaries natively ──────────────────
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-dev-${{ matrix.target }}-
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: weed-volume-normal-${{ matrix.arch }}
build-dev-containers:
needs: [build-rust-binaries]
runs-on: [ubuntu-latest]
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Download pre-built Rust binaries
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-normal-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Docker meta
-
name: Checkout
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v3
with:
images: |
chrislusf/seaweedfs
@@ -116,40 +30,40 @@ jobs:
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
uses: docker/setup-qemu-action@v4.2.0
- name: Create BuildKit config
-
name: Set up QEMU
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
-
name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.6.0
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.6.0
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build
uses: docker/build-push-action@v7
-
name: Build
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
@@ -58,7 +58,7 @@ jobs:
sudo ldconfig
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: go.mod
@@ -126,35 +126,31 @@ jobs:
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v4.2.0
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.6.0
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Determine branch to build
id: branch
env:
INPUT_REF: ${{ inputs.seaweedfs_ref }}
HEAD_REF: ${{ github.head_ref }}
REF_NAME: ${{ github.ref_name }}
run: |
if [ -n "$INPUT_REF" ]; then
echo "branch=$INPUT_REF" >> "$GITHUB_OUTPUT"
if [ -n "${{ inputs.seaweedfs_ref }}" ]; then
echo "branch=${{ inputs.seaweedfs_ref }}" >> "$GITHUB_OUTPUT"
elif [ "${{ github.event_name }}" = "pull_request" ]; then
echo "branch=$HEAD_REF" >> "$GITHUB_OUTPUT"
echo "branch=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
else
echo "branch=$REF_NAME" >> "$GITHUB_OUTPUT"
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push image
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+20 -270
View File
@@ -1,10 +1,9 @@
name: "docker: build latest container"
# Manual fallback only. On tag push, container_release_unified.yml already
# re-tags the released versioned image as `latest` / `latest_large_disk`,
# so a full rebuild here is unnecessary. Run this manually if you need to
# rebuild `latest` from an arbitrary ref.
on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
source_ref:
@@ -24,22 +23,15 @@ on:
- all
- standard
- large_disk
publish:
description: 'Publish images and manifests'
required: true
type: boolean
default: false
permissions:
contents: read
security-events: write
jobs:
setup:
runs-on: ubuntu-latest
outputs:
variants: ${{ steps.set-variants.outputs.variants }}
publish: ${{ steps.set-publish.outputs.publish }}
steps:
- name: Select variants for this run
id: set-variants
@@ -50,89 +42,9 @@ jobs:
variants='["standard","large_disk"]'
fi
echo "variants=$variants" >> "$GITHUB_OUTPUT"
- name: Select publish mode
id: set-publish
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
else
echo "publish=true" >> "$GITHUB_OUTPUT"
fi
# ── Pre-build Rust volume server binaries natively ──────────────────
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v7
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-${{ matrix.target }}-
- name: Build large-disk variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: |
weed-volume-large-disk-${{ matrix.arch }}
weed-volume-normal-${{ matrix.arch }}
build:
needs: [setup, build-rust-binaries]
needs: [setup]
runs-on: ubuntu-latest
strategy:
matrix:
@@ -141,7 +53,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Free Disk Space
@@ -167,35 +79,14 @@ jobs:
if [ "${{ matrix.variant }}" == "large_disk" ]; then
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
else
echo "tag_suffix=" >> $GITHUB_OUTPUT
echo "build_args=" >> $GITHUB_OUTPUT
echo "rust_variant=normal" >> $GITHUB_OUTPUT
fi
- name: Download pre-built Rust binaries
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
@@ -207,7 +98,7 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: matrix.platform != 'amd64'
uses: docker/setup-qemu-action@v4.2.0
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
@@ -215,30 +106,30 @@ jobs:
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.6.0
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.6.0
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
push: ${{ needs.setup.outputs.publish == 'true' }}
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/${{ matrix.platform }}
# Push to GHCR only during build to avoid Docker Hub rate limits
@@ -258,157 +149,16 @@ jobs:
# Remove Go build cache
sudo rm -rf /tmp/go-build*
trivy-scan:
runs-on: ubuntu-latest
needs: [setup, build, build-rust-binaries]
strategy:
matrix:
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
steps:
- name: Configure variant
id: config
run: |
if [ "${{ matrix.variant }}" == "large_disk" ]; then
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
else
echo "tag_suffix=" >> $GITHUB_OUTPUT
fi
- name: Login to GHCR
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4.6.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Checkout for local scan build
if: needs.setup.outputs.publish != 'true'
uses: actions/checkout@v7
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Download pre-built Rust binaries for local scan
if: needs.setup.outputs.publish != 'true'
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context for local scan
if: needs.setup.outputs.publish != 'true'
run: |
rust_variant="normal"
if [ "${{ matrix.variant }}" == "large_disk" ]; then
rust_variant="large-disk"
fi
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
- name: Create BuildKit config for local scan build
if: needs.setup.outputs.publish != 'true'
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx for local scan build
if: needs.setup.outputs.publish != 'true'
uses: docker/setup-buildx-action@v4
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Build local scan image tarball
if: needs.setup.outputs.publish != 'true'
uses: docker/build-push-action@v7
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
file: ./docker/Dockerfile.go_build
platforms: linux/amd64
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
build-args: |
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
- name: Trivy report (published image)
if: needs.setup.outputs.publish == 'true'
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: image
# Scan amd64 only - OS packages are identical across architectures
# since they all use the same alpine base, so a single-arch scan
# provides sufficient coverage without multiplying CI time.
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
- name: Trivy report (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
- name: Upload Trivy scan results to GitHub Security
uses: github/codeql-action/upload-sarif@v4.37.4
if: always()
with:
sarif_file: trivy-results.sarif
- name: Trivy gate (published image)
if: needs.setup.outputs.publish == 'true'
# Gate only on fixable high/critical vulnerabilities. Non-fixable
# findings are still visible in the SARIF upload above.
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: image
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
format: table
exit-code: '1'
skip-setup-trivy: true
- name: Trivy gate (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
format: table
exit-code: '1'
skip-setup-trivy: true
create-manifest:
runs-on: ubuntu-latest
needs: [setup, build, trivy-scan]
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
needs: [setup, build]
if: github.event_name != 'pull_request'
strategy:
matrix:
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
@@ -423,19 +173,19 @@ jobs:
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
- name: Login to Docker Hub
uses: docker/login-action@v4.6.0
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4.6.0
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
@@ -4,19 +4,11 @@ on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag to publish (e.g. 3.93)'
required: true
default: ''
workflow_dispatch: {}
permissions:
contents: read
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
jobs:
build-large-release-container_foundationdb:
@@ -25,16 +17,16 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
tags: |
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
type=ref,event=tag,suffix=_large_disk_foundationdb
flavor: |
latest=false
labels: |
@@ -43,14 +35,14 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@v4.2.0
uses: docker/setup-qemu-action@v3
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4.6.0
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
@@ -65,7 +57,7 @@ jobs:
fi
-
name: Build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+115 -332
View File
@@ -29,11 +29,9 @@ on:
permissions:
contents: read
security-events: write
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
IMAGE: ghcr.io/chrislusf/seaweedfs
# Limit concurrent builds to avoid rate limits
concurrency:
@@ -41,168 +39,86 @@ concurrency:
cancel-in-progress: false
jobs:
# ── Pre-build Rust volume server binaries natively ──────────────────
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
# emulated cargo build into ~15 minutes of native compilation.
build-rust-binaries:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
arch: amd64
- target: aarch64-unknown-linux-musl
arch: arm64
cross: true
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install musl tools (amd64)
if: ${{ !matrix.cross }}
run: sudo apt-get install -y musl-tools
- name: Install cross-compilation tools (arm64)
if: matrix.cross
run: |
sudo apt-get install -y gcc-aarch64-linux-gnu
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-docker-${{ matrix.target }}-
- name: Build large-disk variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
- name: Build normal variant
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.arch }}
path: |
weed-volume-large-disk-${{ matrix.arch }}
weed-volume-normal-${{ matrix.arch }}
# One job per (variant, platform) on a native runner, pushed by digest;
# the merge job stitches the digests into one multi-arch tag.
build:
needs: [build-rust-binaries]
runs-on: ${{ matrix.runner }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
# Build sequentially to avoid rate limits
max-parallel: 2
matrix:
include:
# Normal volume - multi-arch
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
# Large disk - multi-arch
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
# Full tags - multi-arch
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
# Large disk + full tags - multi-arch
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
- variant: normal
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
dockerfile: ./docker/Dockerfile.go_build
build_args: ""
tag_suffix: ""
# Large disk - multi-arch
- variant: large_disk
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset
tag_suffix: _large_disk
# Full tags - amd64 only
- variant: full
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _full
# Large disk + full tags - amd64 only
- variant: large_disk_full
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _large_disk_full
# RocksDB large disk - amd64 only
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
- variant: rocksdb
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.rocksdb_large
build_args: ""
tag_suffix: _large_disk_rocksdb
steps:
- name: Skip unselected variant
if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant
run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0
- name: Checkout
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/checkout@v7
- name: Download pre-built Rust binaries
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/download-artifact@v8
with:
pattern: rust-volume-*
merge-multiple: true
path: ./rust-bins
- name: Place Rust binaries in Docker context
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
mkdir -p docker/weed-volume-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
echo "Placed pre-built Rust binary for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
uses: actions/checkout@v6
- name: Free Disk Space
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
echo "Available disk space before cleanup:"
df -h
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
sudo docker system prune -af --volumes
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
id: docker_meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
with:
images: ${{ env.IMAGE }}
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
flavor: latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu
uses: docker/setup-qemu-action@v4.2.0
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
@@ -210,111 +126,91 @@ jobs:
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.6.0
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build and push ${{ matrix.variant }} (${{ matrix.platform }})
- name: Build and push ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
id: build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platform }}
platforms: ${{ matrix.platforms }}
# Push to GHCR to avoid Docker Hub rate limits on pulls
tags: |
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
labels: ${{ steps.docker_meta.outputs.labels }}
# Flat single-platform manifest so imagetools create assembles cleanly.
provenance: false
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
# max only for rocksdb: its RocksDB compile is sha-independent and worth
# keeping; go-build layers are sha-busted every release, so min elsewhere.
cache-to: type=gha,mode=${{ matrix.variant == 'rocksdb' && 'max' || 'min' }},scope=${{ matrix.variant }}-${{ matrix.arch }}
cache-from: type=gha,scope=${{ matrix.variant }}
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
build-args: |
${{ matrix.build_args }}
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
- name: Export digest
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
- name: Clean up build artifacts
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"
sudo docker system prune -f
sudo rm -rf /tmp/go-build*
- name: Upload digest
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/upload-artifact@v7
with:
name: digest-${{ matrix.variant }}-${{ matrix.arch }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
# Assemble each variant's per-platform digests into one tag, then mirror to Docker Hub.
merge:
needs: [build]
copy-to-dockerhub:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
strategy:
fail-fast: false
matrix:
variant: [normal, large_disk, full, large_disk_full, rocksdb]
include:
- { variant: normal, tag_suffix: "" }
- { variant: large_disk, tag_suffix: _large_disk }
- { variant: full, tag_suffix: _full }
- { variant: large_disk_full, tag_suffix: _large_disk_full }
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb }
- variant: normal
tag_suffix: ""
- variant: large_disk
tag_suffix: _large_disk
- variant: full
tag_suffix: _full
- variant: large_disk_full
tag_suffix: _large_disk_full
- variant: rocksdb
tag_suffix: _large_disk_rocksdb
steps:
- name: Download digests
- name: Login to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/download-artifact@v8
uses: docker/login-action@v3
with:
pattern: digest-${{ matrix.variant }}-*
merge-multiple: true
path: /tmp/digests
- name: Set up Docker Buildx
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/setup-buildx-action@v4
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.6.0
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Create multi-arch tag on GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
working-directory: /tmp/digests
run: |
docker buildx imagetools create \
-t ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
- name: Login to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4.6.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Install crane
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
@@ -322,171 +218,58 @@ jobs:
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Copy ${{ matrix.variant }} to Docker Hub
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
# Function to retry with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Copy multi-arch image from GHCR to Docker Hub with retry
# This is much more efficient than pulling/pushing individual arch images
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
retry_with_backoff crane copy \
${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
echo "Copied ${{ matrix.variant }} to Docker Hub"
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
# Security for visibility, but never blocks the release. Releases (including
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
# we sometimes need to publish through known findings (e.g. unfixed upstream
# CVE, base-image lag).
trivy-scan:
runs-on: ubuntu-latest
needs: [merge]
if: github.event_name == 'push'
continue-on-error: true
strategy:
fail-fast: false
matrix:
include:
- source_suffix: ""
variant: normal
- source_suffix: _large_disk
variant: large_disk
steps:
- name: Login to GHCR
uses: docker/login-action@v4.6.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Trivy report (${{ matrix.variant }})
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: image
# Scan the multi-arch tag on GHCR (already pushed by the build job).
# Trivy scans the runner's native platform; OS packages are identical
# across architectures since they all share the same alpine base.
image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
- name: Upload Trivy scan results to GitHub Security
if: always()
uses: github/codeql-action/upload-sarif@v4.37.4
with:
sarif_file: trivy-results.sarif
category: trivy-${{ matrix.variant }}
# Point `latest` (and `latest_large_disk`) at the just-released versioned
# image. crane tag adds an extra tag to an existing manifest — no rebuild,
# no QEMU, no separate workflow. Replaces the old container_latest.yml
# rebuild that often failed or lagged behind the release. Independent of
# trivy-scan: vuln findings are reported but do not block `latest`.
tag-latest:
runs-on: ubuntu-latest
needs: [merge]
if: github.event_name == 'push'
strategy:
matrix:
include:
- source_suffix: ""
latest_tag: latest
- source_suffix: _large_disk
latest_tag: latest_large_disk
steps:
- name: Login to Docker Hub
uses: docker/login-action@v4.6.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4.6.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
run: |
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
run: |
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
DST_TAG="${{ matrix.latest_tag }}"
echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}"
retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}"
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
helm-release:
runs-on: ubuntu-latest
needs: [build]
needs: [copy-to-dockerhub]
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
permissions:
contents: write
pages: write
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@v1.7.0
with:
with:
token: ${{ secrets.GITHUB_TOKEN }}
charts_dir: k8s/charts
target_dir: helm
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Prepare Docker tag
id: tag
@@ -82,19 +82,19 @@ jobs:
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v1
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
- name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v1
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push image
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: true
+4 -7
View File
@@ -21,22 +21,19 @@ jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'telemetry/server/go.mod'
go-version: '1.24'
- name: Build Telemetry Server
if: github.event_name == 'workflow_dispatch' && inputs.deploy
run: |
# telemetry/server is its own Go module; build from within it
cd telemetry/server
go mod tidy
echo "Building telemetry server..."
GOOS=linux GOARCH=amd64 go build -o ../../telemetry-server .
cd ../..
GOOS=linux GOARCH=amd64 go build -o telemetry-server ./telemetry/server/main.go
ls -la telemetry-server
echo "Build completed successfully"
+2 -2
View File
@@ -9,6 +9,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
- name: 'Dependency Review'
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261
+12 -36
View File
@@ -3,20 +3,8 @@ name: "End to End"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'docker/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/e2e.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'docker/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/e2e.yml'
concurrency:
group: ${{ github.head_ref }}/e2e
@@ -35,24 +23,20 @@ jobs:
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
- name: Set up Go 1.x
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Check out code into the Go module directory
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
@@ -74,12 +58,12 @@ jobs:
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
- name: Start SeaweedFS
timeout-minutes: 15
timeout-minutes: 10
run: |
# Enable Docker buildkit for better caching
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
# Build with retry logic
for i in {1..3}; do
echo "Build attempt $i/3"
@@ -94,18 +78,10 @@ jobs:
sleep 30
fi
done
# Start services with wait
docker compose -f ./compose/e2e-mount.yml up --wait
- name: Rotate buildx cache
if: always()
run: |
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
rm -rf /tmp/.buildx-cache
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
- name: Run FIO 4k
timeout-minutes: 15
run: |
@@ -159,7 +135,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: output-logs
path: docker/output.log
+4 -16
View File
@@ -3,20 +3,8 @@ name: "EC Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'test/erasure_coding/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/ec-integration-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'test/erasure_coding/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/ec-integration-tests.yml'
permissions:
contents: read
@@ -28,13 +16,13 @@ jobs:
timeout-minutes: 30
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Build weed binary
run: |
@@ -64,7 +52,7 @@ jobs:
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ec-integration-test-logs
path: |
+49
View File
@@ -0,0 +1,49 @@
name: EC Integration Tests
on:
push:
branches: [ master ]
paths:
- 'weed/admin/**'
- 'weed/worker/**'
- 'test/erasure_coding/admin_dockertest/**'
- '.github/workflows/ec-integration.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/admin/**'
- 'weed/worker/**'
- 'test/erasure_coding/admin_dockertest/**'
- '.github/workflows/ec-integration.yml'
jobs:
ec-integration-test:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.24'
- name: Build weed binary
run: |
cd weed
go build -o ../weed_bin
- name: Run EC integration tests
run: |
cd test/erasure_coding/admin_dockertest
go test -v -timeout 15m ec_integration_test.go
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v6
with:
name: ec-test-logs
path: test/erasure_coding/admin_dockertest/tmp/logs/
retention-days: 7
@@ -1,68 +0,0 @@
name: "FUSE DLM Integration Tests"
on:
pull_request:
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/cluster/**'
- 'test/fuse_dlm/**'
- '.github/workflows/fuse-dlm-integration.yml'
- '.github/actions/fix-fusermount-setuid/**'
push:
branches: [master]
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/cluster/**'
- 'test/fuse_dlm/**'
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-dlm-integration:
name: FUSE DLM Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install FUSE dependencies
run: |
sudo apt-get update
sudo apt-get install -y libfuse3-dev
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
- name: Repair the fusermount3 setuid bit
uses: ./.github/actions/fix-fusermount-setuid
- name: Build SeaweedFS
run: go build -o weed/weed -buildvcs=false ./weed
- name: Run DLM integration tests
timeout-minutes: 25
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: fuse-dlm-test-logs
path: /tmp/seaweedfs-fuse-dlm-logs/
retention-days: 3
+186 -38
View File
@@ -7,14 +7,12 @@ on:
- 'weed/**'
- 'test/fuse_integration/**'
- '.github/workflows/fuse-integration.yml'
- '.github/actions/fix-fusermount-setuid/**'
pull_request:
branches: [ master, main ]
paths:
- 'weed/**'
- 'test/fuse_integration/**'
- '.github/workflows/fuse-integration.yml'
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.head_ref }}/fuse-integration
@@ -23,64 +21,214 @@ concurrency:
permissions:
contents: read
env:
GO_VERSION: '1.24'
TEST_TIMEOUT: '45m'
jobs:
fuse-integration:
name: FUSE Integration Testing
runs-on: ubuntu-22.04
timeout-minutes: 50
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/checkout@v6
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: ${{ env.GO_VERSION }}
- name: Install FUSE and dependencies
run: |
sudo apt-get update
# fuse3 is pre-installed on ubuntu-22.04 runners and conflicts
# with the legacy fuse package, so only install the dev headers.
sudo apt-get install -y libfuse3-dev
# Allow non-root FUSE mounts with allow_other
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
sudo apt-get install -y fuse libfuse-dev
# Verify FUSE installation
fusermount3 --version || fusermount --version || true
ls -la /dev/fuse
- name: Repair the fusermount3 setuid bit
uses: ./.github/actions/fix-fusermount-setuid
fusermount --version || true
ls -la /dev/fuse || true
- name: Build SeaweedFS
run: |
cd weed
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -o weed .
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
chmod +x weed
# Verify binary
./weed version
# Make weed binary available in PATH for the test framework
sudo cp weed /usr/local/bin/weed
- name: Install test dependencies
- name: Prepare FUSE Integration Tests
run: |
cd test/fuse_integration
go mod download
# Create isolated test directory to avoid Go module conflicts
mkdir -p /tmp/seaweedfs-fuse-tests
# Copy only the working test files to avoid Go module conflicts
# These are the files we've verified work without package name issues
cp test/fuse_integration/simple_test.go /tmp/seaweedfs-fuse-tests/ 2>/dev/null || echo "⚠️ simple_test.go not found"
cp test/fuse_integration/working_demo_test.go /tmp/seaweedfs-fuse-tests/ 2>/dev/null || echo "⚠️ working_demo_test.go not found"
# Note: Other test files (framework.go, basic_operations_test.go, etc.)
# have Go module conflicts and are skipped until resolved
echo "📁 Working test files copied:"
ls -la /tmp/seaweedfs-fuse-tests/*.go 2>/dev/null || echo "ℹ️ No test files found"
# Initialize Go module in isolated directory
cd /tmp/seaweedfs-fuse-tests
go mod init seaweedfs-fuse-tests
go mod tidy
# Verify setup
echo "✅ FUSE integration test environment prepared"
ls -la /tmp/seaweedfs-fuse-tests/
echo ""
echo "ℹ️ Current Status: Running working subset of FUSE tests"
echo " • simple_test.go: Package structure verification"
echo " • working_demo_test.go: Framework capability demonstration"
echo " • Full framework: Available in test/fuse_integration/ (module conflicts pending resolution)"
- name: Run FUSE Integration Tests
run: |
set -o pipefail
cd test/fuse_integration
echo "Running full FUSE integration test suite..."
go test -v -count=1 -timeout=45m ./... 2>&1 | tee /tmp/fuse-test-output.log
- name: Upload Test Logs
cd /tmp/seaweedfs-fuse-tests
echo "🧪 Running FUSE integration tests..."
echo "============================================"
# Run available working test files
TESTS_RUN=0
if [ -f "simple_test.go" ]; then
echo "📋 Running simple_test.go..."
go test -v -timeout=${{ env.TEST_TIMEOUT }} simple_test.go
TESTS_RUN=$((TESTS_RUN + 1))
fi
if [ -f "working_demo_test.go" ]; then
echo "📋 Running working_demo_test.go..."
go test -v -timeout=${{ env.TEST_TIMEOUT }} working_demo_test.go
TESTS_RUN=$((TESTS_RUN + 1))
fi
# Run combined test if multiple files exist
if [ -f "simple_test.go" ] && [ -f "working_demo_test.go" ]; then
echo "📋 Running combined tests..."
go test -v -timeout=${{ env.TEST_TIMEOUT }} simple_test.go working_demo_test.go
fi
if [ $TESTS_RUN -eq 0 ]; then
echo "⚠️ No working test files found, running module verification only"
go version
go mod verify
else
echo "✅ Successfully ran $TESTS_RUN test file(s)"
fi
echo "============================================"
echo "✅ FUSE integration tests completed"
- name: Run Extended Framework Validation
run: |
cd /tmp/seaweedfs-fuse-tests
echo "🔍 Running extended framework validation..."
echo "============================================"
# Test individual components (only run tests that exist)
if [ -f "simple_test.go" ]; then
echo "Testing simple verification..."
go test -v simple_test.go
fi
if [ -f "working_demo_test.go" ]; then
echo "Testing framework demo..."
go test -v working_demo_test.go
fi
# Test combined execution if both files exist
if [ -f "simple_test.go" ] && [ -f "working_demo_test.go" ]; then
echo "Testing combined execution..."
go test -v simple_test.go working_demo_test.go
elif [ -f "simple_test.go" ] || [ -f "working_demo_test.go" ]; then
echo "✅ Individual tests already validated above"
else
echo "⚠️ No working test files found for combined testing"
fi
echo "============================================"
echo "✅ Extended validation completed"
- name: Generate Test Coverage Report
run: |
cd /tmp/seaweedfs-fuse-tests
echo "📊 Generating test coverage report..."
go test -v -coverprofile=coverage.out .
go tool cover -html=coverage.out -o coverage.html
echo "Coverage report generated: coverage.html"
- name: Verify SeaweedFS Binary Integration
run: |
# Test that SeaweedFS binary is accessible from test environment
WEED_BINARY=$(pwd)/weed/weed
if [ -f "$WEED_BINARY" ]; then
echo "✅ SeaweedFS binary found at: $WEED_BINARY"
$WEED_BINARY version
echo "Binary is ready for full integration testing"
else
echo "❌ SeaweedFS binary not found"
exit 1
fi
- name: Upload Test Artifacts
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: fuse-integration-test-results
path: |
/tmp/fuse-test-output.log
/tmp/seaweedfs-fuse-logs/
/tmp/seaweedfs-fuse-tests/coverage.out
/tmp/seaweedfs-fuse-tests/coverage.html
/tmp/seaweedfs-fuse-tests/*.log
retention-days: 7
- name: Test Summary
if: always()
run: |
echo "## 🚀 FUSE Integration Test Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Framework Status" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Framework Design**: Complete and validated" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Working Tests**: Core framework demonstration functional" >> $GITHUB_STEP_SUMMARY
echo "- ⚠️ **Full Framework**: Available but requires Go module resolution" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **CI/CD Integration**: Automated testing pipeline established" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Test Capabilities" >> $GITHUB_STEP_SUMMARY
echo "- 📁 **File Operations**: Create, read, write, delete, permissions" >> $GITHUB_STEP_SUMMARY
echo "- 📂 **Directory Operations**: Create, list, delete, nested structures" >> $GITHUB_STEP_SUMMARY
echo "- 📊 **Large Files**: Multi-megabyte file handling" >> $GITHUB_STEP_SUMMARY
echo "- 🔄 **Concurrent Operations**: Multi-threaded stress testing" >> $GITHUB_STEP_SUMMARY
echo "- ⚠️ **Error Scenarios**: Comprehensive error handling validation" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Comparison with Current Tests" >> $GITHUB_STEP_SUMMARY
echo "| Aspect | Current (FIO) | This Framework |" >> $GITHUB_STEP_SUMMARY
echo "|--------|---------------|----------------|" >> $GITHUB_STEP_SUMMARY
echo "| **Scope** | Performance only | Functional + Performance |" >> $GITHUB_STEP_SUMMARY
echo "| **Operations** | Read/Write only | All FUSE operations |" >> $GITHUB_STEP_SUMMARY
echo "| **Concurrency** | Single-threaded | Multi-threaded stress tests |" >> $GITHUB_STEP_SUMMARY
echo "| **Automation** | Manual setup | Fully automated |" >> $GITHUB_STEP_SUMMARY
echo "| **Validation** | Speed metrics | Correctness + Performance |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Current Working Tests" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Framework Structure**: Package and module verification" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Configuration Management**: Test config validation" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **File Operations Demo**: Basic file create/read/write simulation" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Large File Handling**: 1MB+ file processing demonstration" >> $GITHUB_STEP_SUMMARY
echo "- ✅ **Concurrency Simulation**: Multi-file operation testing" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
echo "1. **Module Resolution**: Fix Go package conflicts for full framework" >> $GITHUB_STEP_SUMMARY
echo "2. **SeaweedFS Integration**: Connect with real cluster for end-to-end testing" >> $GITHUB_STEP_SUMMARY
echo "3. **Performance Benchmarks**: Add performance regression testing" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "📈 **Total Framework Size**: ~1,500 lines of comprehensive testing infrastructure" >> $GITHUB_STEP_SUMMARY
@@ -1,74 +0,0 @@
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
on:
pull_request:
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/filer/mount_peer_registry*.go'
- 'weed/server/filer_grpc_server_mount_peer.go'
- 'weed/pb/mount_peer.proto'
- 'weed/pb/filer.proto'
- 'test/fuse_p2p/**'
- '.github/workflows/fuse-p2p-integration.yml'
- '.github/actions/fix-fusermount-setuid/**'
push:
branches: [master]
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/filer/mount_peer_registry*.go'
- 'weed/server/filer_grpc_server_mount_peer.go'
- 'weed/pb/mount_peer.proto'
- 'weed/pb/filer.proto'
- 'test/fuse_p2p/**'
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-p2p-integration:
name: FUSE P2P Peer Chunk Sharing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install FUSE dependencies
run: |
sudo apt-get update
sudo apt-get install -y libfuse3-dev
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
- name: Repair the fusermount3 setuid bit
uses: ./.github/actions/fix-fusermount-setuid
- name: Build SeaweedFS
run: go build -o weed/weed -buildvcs=false ./weed
- name: Run P2P integration tests
timeout-minutes: 15
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: fuse-p2p-test-logs
path: /tmp/seaweedfs-fuse-p2p-logs/
retention-days: 3
+18 -89
View File
@@ -3,18 +3,8 @@ name: "go: build binary"
on:
push:
branches: [ master ]
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- '.github/workflows/go.yml'
pull_request:
branches: [ master ]
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- '.github/workflows/go.yml'
concurrency:
group: ${{ github.head_ref }}/go
@@ -29,12 +19,13 @@ jobs:
name: Go Vet
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
- name: Set up Go 1.x
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Get dependencies
run: |
cd weed; go get -v -t -d ./...
@@ -47,92 +38,30 @@ jobs:
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
if grep -q "vet:" vet-output.txt; then exit 1; fi
vet-32bit:
name: Go Vet 32-bit
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
run: |
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
if grep -q "vet:" vet-32bit-output.txt; then exit 1; fi
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
- name: Set up Go 1.x
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
build-cross:
name: Build cross-platform (${{ matrix.goos }}/${{ matrix.goarch }})
runs-on: ubuntu-latest
strategy:
# One target's breakage should not hide the other three.
fail-fast: false
matrix:
include:
- { goos: windows, goarch: amd64 }
- { goos: windows, goarch: arm64 }
- { goos: freebsd, goarch: amd64 }
- { goos: darwin, goarch: arm64 }
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
# Nothing else on a PR compiles these, so per-OS syscall constants creep
# back into shared files unnoticed and only a release build catches it.
- name: Cross-compile
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
run: |
go build ./weed/...
# Tests too: they reach for per-OS syscall constants the build does
# not, and only compiling them catches an untagged one.
go vet ./weed/mount/... ./weed/command/... 2>&1 |
grep -v "MessageState contains sync.Mutex" | tee /tmp/vet.txt
if grep -q "vet:" /tmp/vet.txt; then exit 1; fi
test:
name: Test
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
- name: Set up Go 1.x
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Test
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
test-32bit:
name: Test 32-bit
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
# 386 test binaries run natively on the amd64 runner. This catches what vet
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
# -short skips the e2e suites already covered on amd64.
- name: Test linux/386
run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
File diff suppressed because it is too large Load Diff
+3 -19
View File
@@ -1,25 +1,21 @@
name: "helm: release"
name: "helm: manual release"
on:
push:
tags:
- '*'
workflow_dispatch:
permissions:
contents: write
pages: write
packages: write
jobs:
helm-release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Publish Helm charts to github pages
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@v1.7.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
@@ -27,15 +23,3 @@ jobs:
target_dir: helm
branch: gh-pages
helm_version: "3.18.4"
- name: Publish Helm charts to github container registry (ghcr.io)
uses: bitdeps/helm-oci-charts-releaser@v0.1.5
with:
charts_dir: k8s/charts
github_token: ${{ secrets.GITHUB_TOKEN }}
oci_registry: ghcr.io/${{ github.repository_owner }}
oci_username: github-actions
oci_password: ${{ secrets.GITHUB_TOKEN }}
skip_dependencies: true
skip_helm_install: true
skip_gh_release: true
+3 -3
View File
@@ -25,10 +25,10 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -111,7 +111,7 @@ jobs:
# Wait for S3 API
for i in {1..30}; do
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
echo "✓ S3 API is ready"
break
fi
-183
View File
@@ -1,183 +0,0 @@
name: "release: java clients"
# Publishes the SeaweedFS Java clients (seaweedfs-client and
# seaweedfs-hadoop3-client) to Maven Central via the Sonatype Central Portal.
#
# Required repository secrets:
# MAVEN_CENTRAL_USERNAME - Central Portal user token username (central.sonatype.com -> Account -> Generate User Token)
# MAVEN_CENTRAL_PASSWORD - Central Portal user token password
# MAVEN_GPG_PRIVATE_KEY - ASCII-armored signing key:
# gpg --homedir <keyring> --armor --export-secret-keys <KEYID> > key.asc
# MAVEN_GPG_PASSPHRASE - passphrase for that key
#
# A plain run publishes to Central. Check dry_run to exercise secrets, GPG
# signing, and the build without committing or publishing.
on:
workflow_dispatch:
inputs:
version:
description: "Version to publish, e.g. 4.39. Leave blank to use the current SeaweedFS version."
type: string
required: false
dry_run:
description: "Build and GPG-sign only; skip the version commit and the Central upload"
type: boolean
default: false
permissions:
contents: write
concurrency:
group: java-release
cancel-in-progress: false
env:
# Java 17 needs these opens for the Maven publishing plugins' reflective access.
MAVEN_OPTS: >-
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.text=ALL-UNNAMED
--add-opens=java.desktop/java.awt.font=ALL-UNNAMED
jobs:
publish:
name: Publish to Maven Central
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
with:
# Full history so the version-bump commit can rebase onto a moved master.
fetch-depth: 0
# An explicit input wins; otherwise fall back to the current SeaweedFS
# version (MAJOR.MINOR) from constants.go, matching its %d.%02d formatting.
- name: Resolve version
id: resolve
env:
INPUT_VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
VERSION="$INPUT_VERSION"
if [ -z "$VERSION" ]; then
CONST=weed/util/version/constants.go
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
echo "No version given; using current SeaweedFS version ${VERSION}."
fi
if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
echo "::error::version must be MAJOR.MINOR, e.g. 4.39 (got '$VERSION')"
exit 1
fi
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
- name: Set up JDK 17
uses: actions/setup-java@v5
with:
java-version: '17'
distribution: 'temurin'
cache: 'maven'
server-id: central
server-username: MAVEN_CENTRAL_USERNAME
server-password: MAVEN_CENTRAL_PASSWORD
gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
# client carries the version literally; hdfs3 derives its own version and
# its seaweedfs-client dependency from the seaweedfs.client.version property.
- name: Set versions in poms
env:
VERSION: ${{ steps.resolve.outputs.version }}
run: |
set -euo pipefail
V=org.codehaus.mojo:versions-maven-plugin:2.16.2
mvn -B -ntp -f other/java/client/pom.xml "${V}:set" \
-DnewVersion="$VERSION" -DgenerateBackupPoms=false
mvn -B -ntp -f other/java/hdfs3/pom.xml "${V}:set-property" \
-Dproperty=seaweedfs.client.version -DnewVersion="$VERSION" -DgenerateBackupPoms=false
- name: Commit version bump
if: ${{ !inputs.dry_run }}
env:
VERSION: ${{ steps.resolve.outputs.version }}
run: |
set -euo pipefail
if git diff --quiet; then
echo "Poms already at ${VERSION}; nothing to commit."
exit 0
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add other/java/client/pom.xml other/java/hdfs3/pom.xml
git commit -m "java ${VERSION}"
# Rebase and retry so a concurrent push to master doesn't lose the bump.
for attempt in 1 2 3 4 5; do
if git push origin HEAD:master; then
exit 0
fi
echo "push rejected (attempt ${attempt}); rebasing onto latest master"
git pull --rebase origin master
done
echo "::error::could not push version bump after retries"
exit 1
# Tests are skipped here: the integration tests need a live filer, and the
# offline unit tests already run on every push in java_unit_tests.yml.
# dry_run stops at `install` (build + GPG sign, no upload). A real run deploys.
# client goes first either way so its install populates the local repo for hdfs3.
- name: Publish seaweedfs-client
working-directory: other/java/client
env:
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
DRY_RUN: ${{ inputs.dry_run }}
run: |
set -euo pipefail
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
mvn -B -ntp $GOAL -DskipTests
- name: Publish seaweedfs-hadoop3-client
working-directory: other/java/hdfs3
env:
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
DRY_RUN: ${{ inputs.dry_run }}
run: |
set -euo pipefail
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
mvn -B -ntp $GOAL -DskipTests
# Bump every version the wiki quotes for the Java clients. Runs only after
# a successful real publish. Uses GITHUB_TOKEN; set a WIKI_TOKEN secret if
# the token can't push to the wiki.
- name: Update wiki client versions
if: ${{ !inputs.dry_run }}
env:
VERSION: ${{ steps.resolve.outputs.version }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WIKI_TOKEN: ${{ secrets.WIKI_TOKEN }}
run: |
set -euo pipefail
TOKEN="${WIKI_TOKEN:-$GH_TOKEN}"
git clone -q "https://x-access-token:${TOKEN}@github.com/${{ github.repository }}.wiki.git" wiki
cd wiki
for f in $(grep -rlE "seaweedfs-(client|hadoop3-client)" --include="*.md" .); do
NEWV="$VERSION" perl -0777 -i -pe '
my $v = $ENV{NEWV};
s{(<artifactId>seaweedfs-(?:client|hadoop3-client)</artifactId>\s*<version>)[0-9]+\.[0-9]+(</version>)}{$1$v$2}g;
s{(seaweedfs-(?:client|hadoop3-client):)[0-9]+\.[0-9]+}{$1$v}g;
s{(seaweedfs-(?:client|hadoop3-client)-)[0-9]+\.[0-9]+(\.jar)}{$1$v$2}g;
s{(seaweedfs-(?:client|hadoop3-client)/)[0-9]+\.[0-9]+(/)}{$1$v$2}g;
' "$f"
done
if git diff --quiet; then
echo "Wiki already references ${VERSION}."
exit 0
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git commit -am "java clients ${VERSION}"
git push
+2 -2
View File
@@ -23,7 +23,7 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Java
uses: actions/setup-java@v5
@@ -49,7 +49,7 @@ jobs:
- name: Upload Test Reports
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: test-reports-java-${{ matrix.java }}
path: |
+4 -25
View File
@@ -3,24 +3,8 @@ name: "Kafka Quick Test (Load Test with Schema Registry)"
on:
push:
branches: [ master ]
paths:
- 'weed/mq/**'
- 'weed/pb/mq_pb/**'
- 'weed/pb/schema_pb/**'
- 'test/kafka/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/kafka-quicktest.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/mq/**'
- 'weed/pb/mq_pb/**'
- 'weed/pb/schema_pb/**'
- 'test/kafka/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/kafka-quicktest.yml'
workflow_dispatch: # Allow manual trigger
concurrency:
@@ -37,24 +21,19 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Install dependencies
run: |
+22 -38
View File
@@ -3,24 +3,8 @@ name: "Kafka Gateway Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/mq/**'
- 'weed/pb/mq_pb/**'
- 'weed/pb/schema_pb/**'
- 'test/kafka/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/kafka-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/mq/**'
- 'weed/pb/mq_pb/**'
- 'weed/pb/schema_pb/**'
- 'test/kafka/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/kafka-tests.yml'
concurrency:
group: ${{ github.head_ref }}/kafka-tests
@@ -51,13 +35,13 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Setup Container Environment
run: |
@@ -96,13 +80,13 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Setup Integration Container Environment
run: |
@@ -143,12 +127,12 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
@@ -322,12 +306,12 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
@@ -461,7 +445,7 @@ jobs:
# Test consumer group functionality with explicit timeout
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
ulimit -u 100 || echo "Warning: Could not set process limit"
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
env:
GOMAXPROCS: 1
SEAWEEDFS_MASTERS: 127.0.0.1:9333
@@ -484,12 +468,12 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
@@ -642,12 +626,12 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
@@ -803,13 +787,13 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Setup Protocol Container Environment
run: |
-140
View File
@@ -1,140 +0,0 @@
name: "KMS Tests"
on:
pull_request:
paths:
- 'weed/kms/**'
- 'weed/s3api/s3_sse_*.go'
- 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'test/kms/**'
- '.github/workflows/kms-tests.yml'
push:
branches: [ master, main ]
paths:
- 'weed/kms/**'
- 'weed/s3api/s3_sse_*.go'
- 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'test/kms/**'
concurrency:
group: ${{ github.head_ref || github.ref }}-kms-tests
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
working-directory: weed
jobs:
kms-provider-tests:
name: KMS Provider Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run KMS provider integration tests
timeout-minutes: 15
working-directory: test/kms
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
docker --version
make test-provider-ci
- name: Show OpenBao logs on failure
if: failure()
run: |
echo "=== OpenBao Container Logs ==="
docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
echo "=== Setup Logs ==="
cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found"
- name: Cleanup
if: always()
working-directory: test/kms
run: |
make stop-openbao-ci || true
s3-kms-e2e-tests:
name: S3 KMS End-to-End Tests
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 KMS end-to-end tests
timeout-minutes: 20
working-directory: test/kms
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
docker --version
aws --version
make test-s3-kms-ci
- name: Show logs on failure
if: failure()
working-directory: test/kms
run: |
echo "=== OpenBao Container Logs ==="
cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
echo "=== SeaweedFS Server Logs ==="
tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
echo "=== Setup Logs ==="
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
echo "=== Process Information ==="
ps aux | grep -E "(weed|test)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-kms-e2e-logs
path: |
/tmp/seaweedfs-kms-mini.log
/tmp/openbao-ci-container.log
/tmp/weed-kms-ci-setup.log
retention-days: 3
- name: Cleanup
if: always()
working-directory: test/kms
run: |
make stop-seaweedfs-ci || true
make stop-openbao-ci || true
make clean-ci || true
@@ -1,79 +0,0 @@
name: "Master Cold Start Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/topology/**'
- 'weed/operation/**'
- 'test/master_cold_start/**'
- 'test/testutil/**'
- '.github/workflows/master-cold-start-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/topology/**'
- 'weed/operation/**'
- 'test/master_cold_start/**'
- 'test/testutil/**'
- '.github/workflows/master-cold-start-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/master-cold-start-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
master-cold-start-tests:
name: Master Cold Start Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v7
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Build weed binary
run: |
cd weed && go install -buildvcs=false
- name: Run master cold start tests
# test/master_cold_start boots a fresh master plus empty volume
# servers and requires the very first assign (HTTP and gRPC, no
# client retries) to complete a write: the assign that triggers
# volume growth must wait for it instead of failing with
# "volume growth in progress".
run: |
export WEED_BINARY=$(go env GOPATH)/bin/weed
go test -v -timeout=8m ./test/master_cold_start/...
- name: Collect server logs on failure
if: failure()
run: |
# test/master_cold_start/cluster.go keeps failing-test dirs created
# via os.MkdirTemp("", "seaweedfs_master_cold_start_it_") with each
# process log under <baseDir>/logs/.
mkdir -p /tmp/master-cold-start-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_master_cold_start_it_*" 2>/dev/null | while read dir; do
echo "Found test directory: $dir"
cp -r "$dir" /tmp/master-cold-start-logs/ 2>/dev/null || true
done
find /tmp/master-cold-start-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: master-cold-start-test-logs
path: /tmp/master-cold-start-logs/
retention-days: 7
@@ -30,6 +30,7 @@ permissions:
contents: read
env:
GO_VERSION: '1.24'
TEST_TIMEOUT: '10m'
jobs:
@@ -40,12 +41,12 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: ${{ env.GO_VERSION }}
- name: Build SeaweedFS
run: |
@@ -69,7 +70,7 @@ jobs:
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: metadata-subscribe-test-logs
path: |
@@ -1,124 +0,0 @@
name: "mount: windows conformance"
on:
push:
branches: [ master ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*.go'
- 'weed/storage/volume_vacuum*.go'
- 'weed/storage/volume_loading.go'
- 'weed/storage/disk_location.go'
- 'test/winfsp-conformance/**'
- '.github/workflows/mount-windows-conformance.yml'
# No base branch filter: this is the only thing that runs the Windows mount,
# so it should cover a pull request stacked on another one too.
pull_request:
paths:
- 'weed/mount/**'
- 'weed/command/mount*.go'
- 'weed/storage/volume_vacuum*.go'
- 'weed/storage/volume_loading.go'
- 'weed/storage/disk_location.go'
- 'test/winfsp-conformance/**'
- '.github/workflows/mount-windows-conformance.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
jobs:
conformance:
name: WinFsp conformance
runs-on: windows-latest
timeout-minutes: 60
env:
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
CGO_ENABLED: 0
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
# not to build.
- name: Install WinFsp
run: choco install winfsp -y --no-progress
- name: Build weed.exe
run: go build -o weed.exe ./weed
# The runner tears down a step's process tree when its shell exits, so a
# cluster started in one step is gone by the next. Everything that needs
# the cluster and the mount alive has to share a step.
- name: Mount and run winfsp-tests
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
function Test-Port($port) {
# A plain connect, because Test-NetConnection has reported success
# here for a port nothing was listening on.
$client = New-Object System.Net.Sockets.TcpClient
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
catch { return $false }
finally { $client.Dispose() }
}
function Start-Mount($log) {
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
$deadline = (Get-Date).AddMinutes(2)
while ((Get-Date) -lt $deadline) {
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
Start-Sleep -Seconds 2
}
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
throw "S: never appeared"
}
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
# -ip pins the cluster to loopback; it otherwise advertises and binds
# the runner's LAN address, which 127.0.0.1 cannot reach.
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
$deadline = (Get-Date).AddMinutes(3)
while ((Get-Date) -lt $deadline) {
# The mount dials grpc, not http, so both ports have to answer.
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
Start-Sleep -Seconds 3
}
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
throw "filer never came up"
}
Write-Host "filer is up on http 8888 and grpc 18888"
Start-Mount 'seaweed-mount'
Write-Host "::group::winfsp-tests"
& pwsh -File test/winfsp-conformance/run.ps1 -MountPoint S:\
$code = $LASTEXITCODE
Write-Host "::endgroup::"
if ($code -ne 0) { throw "winfsp-tests failed with exit $code" }
- name: Logs
if: always()
shell: pwsh
run: |
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
}
-194
View File
@@ -1,194 +0,0 @@
name: "mount: windows"
on:
push:
branches: [ master ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*.go'
- 'weed/storage/volume_vacuum*.go'
- 'weed/storage/volume_loading.go'
- 'weed/storage/disk_location.go'
- 'test/winfsp/**'
- '.github/workflows/mount-windows.yml'
# No base branch filter: this is the only thing that runs the Windows mount,
# so it should cover a pull request stacked on another one too.
pull_request:
paths:
- 'weed/mount/**'
- 'weed/command/mount*.go'
- 'weed/storage/volume_vacuum*.go'
- 'weed/storage/volume_loading.go'
- 'weed/storage/disk_location.go'
- 'test/winfsp/**'
- '.github/workflows/mount-windows.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
jobs:
mount-windows:
name: Mount on Windows
runs-on: windows-latest
timeout-minutes: 40
env:
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
CGO_ENABLED: 0
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
# not to build.
- name: Install WinFsp
run: choco install winfsp -y --no-progress
- name: Build weed.exe
run: go build -o weed.exe ./weed
# The runner tears down a step's process tree when its shell exits, so a
# cluster started in one step is gone by the next. Everything that needs
# the cluster and the mount alive has to share a step.
- name: Mount and exercise
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
function Test-Port($port) {
# A plain connect, because Test-NetConnection has reported success
# here for a port nothing was listening on.
$client = New-Object System.Net.Sockets.TcpClient
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
catch { return $false }
finally { $client.Dispose() }
}
function Start-Mount($log) {
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
$deadline = (Get-Date).AddMinutes(2)
while ((Get-Date) -lt $deadline) {
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
Start-Sleep -Seconds 2
}
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
throw "S: never appeared"
}
function Stop-Mount {
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
Where-Object { $_.CommandLine -like '*mount*' } |
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
$deadline = (Get-Date).AddMinutes(1)
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
if (Test-Path S:\) { throw "S: still present after stopping the mount" }
Write-Host "unmounted"
}
function Invoke-Tests($label, [string[]]$goArgs) {
Write-Host "::group::$label"
& go @goArgs
$code = $LASTEXITCODE
Write-Host "::endgroup::"
if ($code -ne 0) { throw "$label failed with exit $code" }
}
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
# -ip pins the cluster to loopback; it otherwise advertises and binds
# the runner's LAN address, which 127.0.0.1 cannot reach.
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
$deadline = (Get-Date).AddMinutes(3)
while ((Get-Date) -lt $deadline) {
# The mount dials grpc, not http, so both ports have to answer.
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
Start-Sleep -Seconds 3
}
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
throw "filer never came up"
}
Write-Host "filer is up on http 8888 and grpc 18888"
Start-Mount 'seaweed-mount'
Invoke-Tests 'exercise' @('test','-v','-timeout','20m','./test/winfsp','-mountpoint=S:\')
Invoke-Tests 'persist-write' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=write','-filer=127.0.0.1:8888')
Stop-Mount
Start-Mount 'seaweed-remount'
Invoke-Tests 'persist-verify' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=verify')
# WinFsp creates the mount directory itself, so the path must not
# exist; only its parent has to.
Write-Host "::group::mount over a directory"
Stop-Mount
Remove-Item C:\seaweed-mnt -Recurse -Force -ErrorAction SilentlyContinue
Start-Process -FilePath .\weed.exe `
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=C:\seaweed-mnt' `
-RedirectStandardOutput C:\seaweed-dirmount.log -RedirectStandardError C:\seaweed-dirmount.err.log
$deadline = (Get-Date).AddMinutes(2)
$ok = $false
while ((Get-Date) -lt $deadline) {
# Listing succeeds on the plain empty directory too, so wait for the
# reparse point WinFsp turns it into. Otherwise this step passes
# without a mount and writes to local disk.
$item = Get-Item C:\seaweed-mnt -Force -ErrorAction SilentlyContinue
if ($null -ne $item -and $item.Attributes.ToString() -like '*ReparsePoint*') { $ok = $true; break }
Start-Sleep -Seconds 2
}
if (-not $ok) {
Get-Content C:\seaweed-dirmount.log, C:\seaweed-dirmount.err.log -ErrorAction SilentlyContinue
throw "mounting over a directory failed"
}
Set-Content -Path C:\seaweed-mnt\dirmount.txt -Value 'via directory mount'
if ((Get-Content C:\seaweed-mnt\dirmount.txt) -ne 'via directory mount') { throw "readback through the directory mount differs" }
Remove-Item C:\seaweed-mnt\dirmount.txt -Force
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
Where-Object { $_.CommandLine -like '*mount*' } |
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
Start-Sleep -Seconds 5
Write-Host "::endgroup::"
Start-Mount 'seaweed-remount2'
Write-Host "::group::explorer-style walk"
New-Item -ItemType Directory -Force -Path S:\walk | Out-Null
1..200 | ForEach-Object { Set-Content -Path "S:\walk\f$_.txt" -Value "line $_" }
$names = @(Get-ChildItem S:\walk | ForEach-Object { $_.Name })
if ($names.Count -ne 200) {
# Name the strays: a dot entry surfacing here is a different problem
# from a missing or duplicated file.
$unexpected = $names | Where-Object { $_ -notmatch '^f\d+\.txt$' }
throw "listed $($names.Count) entries, expected 200; unexpected: $($unexpected -join ', ')"
}
$body = Get-Content S:\walk\f42.txt
if ($body -ne 'line 42') { throw "unexpected content: $body" }
Copy-Item S:\walk\f42.txt S:\walk\copy.txt
Remove-Item S:\walk -Recurse -Force
if (Test-Path S:\walk) { throw "directory survived recursive delete" }
Write-Host "::endgroup::"
- name: Logs
if: always()
shell: pwsh
run: |
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-remount.log','C:\seaweed-remount.err.log','C:\seaweed-remount2.log','C:\seaweed-remount2.err.log','C:\seaweed-dirmount.log','C:\seaweed-dirmount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
}
-77
View File
@@ -1,77 +0,0 @@
name: "Multi-Master Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/server/raft_*.go'
- 'weed/topology/**'
- 'test/multi_master/**'
- 'test/testutil/**'
- '.github/workflows/multi-master-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/server/raft_*.go'
- 'weed/topology/**'
- 'test/multi_master/**'
- 'test/testutil/**'
- '.github/workflows/multi-master-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/multi-master-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
multi-master-failover-tests:
name: Multi-Master Failover Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Run multi-master failover tests
# The tests in test/multi_master spin up their own 3-node master raft
# cluster (using the freshly-installed `weed` binary) and exercise
# leader-election, failover and recovery scenarios. The shared
# test/testutil port-allocator regression test runs alongside since it
# is a prerequisite for the cluster fixtures.
run: |
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
- name: Collect server logs on failure
if: failure()
run: |
# test/multi_master/cluster.go creates per-test dirs via
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
# node's log into <baseDir>/logs/master*.log.
echo "Collecting per-node master logs from temp directories..."
mkdir -p /tmp/multi-master-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
echo "Found test directory: $dir"
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
done
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: multi-master-test-logs
path: /tmp/multi-master-logs/
retention-days: 7
-477
View File
@@ -1,477 +0,0 @@
name: "Performance"
on:
push:
branches: [ master ]
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- 'seaweed-volume/**'
- 'test/perf/**'
- '.github/workflows/performance.yml'
workflow_dispatch:
inputs:
profile_duration:
description: "CPU profiling duration in seconds"
required: false
default: "30"
type: string
benchmark_files:
description: "Number of files for the throughput benchmark"
required: false
default: "100000"
type: string
benchmark_concurrency:
description: "Concurrent read/write workers"
required: false
default: "16"
type: string
benchmark_size:
description: "Simulated file size in bytes"
required: false
default: "1024"
type: string
s3_objects:
description: "Number of objects for the S3 benchmark"
required: false
default: "20000"
type: string
s3_size:
description: "S3 object size in bytes"
required: false
default: "4096"
type: string
concurrency:
group: ${{ github.head_ref || github.ref }}/performance
cancel-in-progress: true
permissions:
contents: read
env:
VOL_SIZE_LIMIT: "1024"
jobs:
performance-profile:
name: CPU and Heap Profile
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Build weed
run: go build -o weed_bin ./weed
- name: Start server with profiling enabled
run: |
mkdir -p ./perfdata
./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \
-s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \
-s3.port=8000 -s3.config=./docker/compose/s3.json \
> weed.log 2>&1 &
echo "WEED_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
break
fi
sleep 1
done
# give the volume server a moment to register with the master
sleep 3
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Capture profiles under load
run: |
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
# drive write load so the sampled profile reflects real work
./weed_bin benchmark -master=localhost:9333 -writeOnly \
-c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 &
echo "Sampling CPU profile for ${DURATION}s..."
curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof
curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof
curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS)"
echo '```'
if [ -f mem-profile.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Profile summary
if: always()
run: |
{
echo "## CPU profile (top functions)"
echo '```'
head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Stop server
if: always()
run: kill "${WEED_PID}" 2>/dev/null || true
- name: Show server log on failure
if: failure()
run: tail -200 weed.log || true
- name: Upload profiles
if: always()
uses: actions/upload-artifact@v7
with:
name: performance-profile-${{ github.run_number }}
path: |
cpu.pprof
heap.pprof
cpu-top.txt
heap-top.txt
goroutine.txt
mem-profile.csv
mem-profile.csv.peak
weed.log
retention-days: 30
benchmark:
name: Throughput Benchmark (${{ matrix.impl }} volume)
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
impl: [go, rust]
env:
IMPL: ${{ matrix.impl }}
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
if: matrix.impl == 'rust'
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust'
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build weed
run: go build -o weed_bin ./weed
- name: Build Rust volume server
if: matrix.impl == 'rust'
run: cd seaweed-volume && cargo build --release
- name: Start master and volume server
run: |
mkdir -p ./perfdata/master ./perfdata/vol
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
-mdir=./perfdata/master -peers=none \
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
> master.log 2>&1 &
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
break
fi
sleep 1
done
if [ "${IMPL}" = "rust" ]; then
./seaweed-volume/target/release/weed-volume \
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
> volume.log 2>&1 &
else
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
-port=8080 -dir=./perfdata/vol -max=100 \
> volume.log 2>&1 &
fi
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
echo "volume server is ready"
break
fi
sleep 1
done
# let the volume server register with the master via heartbeat
sleep 3
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-benchmark.csv \
"master=${MASTER_PID}" "volume=${VOLUME_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Run throughput benchmark
run: |
N="${{ github.event.inputs.benchmark_files || '100000' }}"
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
SIZE="${{ github.event.inputs.benchmark_size || '1024' }}"
./weed_bin benchmark -master=localhost:9333 \
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
- name: Run Go micro-benchmarks
if: matrix.impl == 'go'
continue-on-error: true
run: |
go test -run='^$' -bench=. -benchmem -benchtime=10x \
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
2>&1 | tee go-benchmarks.txt
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS, ${IMPL} volume)"
echo '```'
if [ -f mem-benchmark.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Benchmark summary
if: always()
run: |
{
echo "## Throughput benchmark (${IMPL} volume)"
echo '```'
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Stop processes
if: always()
run: |
kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
- name: Show logs on failure
if: failure()
run: |
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
- name: Upload benchmark results
if: always()
uses: actions/upload-artifact@v7
with:
name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
path: |
benchmark-results.txt
go-benchmarks.txt
mem-benchmark.csv
mem-benchmark.csv.peak
master.log
volume.log
retention-days: 7
s3-benchmark:
name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
impl: [go, rust]
env:
IMPL: ${{ matrix.impl }}
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
if: matrix.impl == 'rust'
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust'
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build weed and S3 load tool
run: |
go build -o weed_bin ./weed
go build -o s3bench ./test/s3/benchmark
- name: Build Rust volume server
if: matrix.impl == 'rust'
run: cd seaweed-volume && cargo build --release
- name: Start cluster with S3 gateway
run: |
mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
-mdir=./perfdata/master -peers=none \
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
> master.log 2>&1 &
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
break
fi
sleep 1
done
if [ "${IMPL}" = "rust" ]; then
./seaweed-volume/target/release/weed-volume \
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
> volume.log 2>&1 &
else
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
-port=8080 -dir=./perfdata/vol -max=100 \
> volume.log 2>&1 &
fi
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
echo "volume server is ready"
break
fi
sleep 1
done
sleep 3
./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \
-s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \
> filer.log 2>&1 &
echo "FILER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 30); do
if nc -z localhost 8000 2>/dev/null; then
echo "s3 gateway is ready"
break
fi
sleep 1
done
sleep 2
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-s3.csv \
"master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Run S3 read/write benchmark
run: |
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
SIZE="${{ github.event.inputs.s3_size || '4096' }}"
./s3bench -endpoint=http://localhost:8000 \
-access-key=some_access_key1 -secret-key=some_secret_key1 \
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
2>&1 | tee s3-benchmark-results.txt
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS, ${IMPL} volume)"
echo '```'
if [ -f mem-s3.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: S3 benchmark summary
if: always()
run: |
{
echo "## S3 read/write benchmark (${IMPL} volume)"
echo '```'
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Stop processes
if: always()
run: |
kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
- name: Show logs on failure
if: failure()
run: |
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true
- name: Upload S3 benchmark results
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
path: |
s3-benchmark-results.txt
mem-s3.csv
mem-s3.csv.peak
master.log
volume.log
filer.log
retention-days: 7
-118
View File
@@ -1,118 +0,0 @@
name: "pjdfstest POSIX Compliance"
on:
push:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'test/pjdfstest/**'
- '.github/workflows/pjdfstest.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'test/pjdfstest/**'
- '.github/workflows/pjdfstest.yml'
workflow_dispatch:
concurrency:
group: pjdfstest/${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
pjdfstest:
name: pjdfstest
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Start local Docker registry
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Build weed race binary
run: |
cd docker
make binary_race
- name: Build SeaweedFS e2e image
uses: docker/build-push-action@v7
with:
context: docker
file: docker/Dockerfile.e2e
tags: localhost:5000/chrislusf/seaweedfs:e2e
push: true
cache-from: type=gha,scope=pjdfstest-e2e
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
- name: Tag e2e image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:e2e
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
- name: Build pjdfstest image
uses: docker/build-push-action@v7
with:
context: test/pjdfstest
build-contexts: |
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
push: true
cache-from: type=gha,scope=pjdfstest-harness
cache-to: type=gha,mode=max,scope=pjdfstest-harness
- name: Tag pjdfstest image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
- name: Start SeaweedFS cluster
run: |
docker compose -f test/pjdfstest/docker-compose.yml up --wait
- name: Run pjdfstest
run: |
set -o pipefail
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
- name: Collect logs
if: always()
run: |
mkdir -p /tmp/pjdfstest-docker-logs
for svc in master volume filer mount; do
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
done
- name: Tear down
if: always()
run: |
docker compose -f test/pjdfstest/docker-compose.yml down -v
- name: Upload logs
if: always()
uses: actions/upload-artifact@v7
with:
name: pjdfstest-results
path: |
/tmp/pjdfstest-output.log
/tmp/pjdfstest-docker-logs/
retention-days: 7
-51
View File
@@ -1,51 +0,0 @@
name: "Plugin Worker Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'test/plugin_workers/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/plugin-workers.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'test/plugin_workers/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/plugin-workers.yml'
permissions:
contents: read
jobs:
plugin-worker:
name: "Plugin Worker: ${{ matrix.worker }}"
runs-on: ubuntu-22.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
include:
- worker: erasure_coding
path: test/plugin_workers/erasure_coding
- worker: vacuum
path: test/plugin_workers/vacuum
- worker: volume_balance
path: test/plugin_workers/volume_balance
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
go-version: ^1.26
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Run plugin worker tests
run: go test -v ./${{ matrix.path }}
+6 -27
View File
@@ -3,24 +3,8 @@ name: "PostgreSQL Gateway Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/server/postgres/**'
- 'weed/query/**'
- 'weed/mq/**'
- 'test/postgres/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/postgres-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/server/postgres/**'
- 'weed/query/**'
- 'weed/mq/**'
- 'test/postgres/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/postgres-tests.yml'
concurrency:
group: ${{ github.head_ref }}/postgres-tests
@@ -39,24 +23,19 @@ jobs:
working-directory: test/postgres
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code
uses: actions/checkout@v7
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
uses: actions/checkout@v6
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
@@ -83,7 +62,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: postgres-logs
path: test/postgres/postgres-output.log
-115
View File
@@ -1,115 +0,0 @@
name: "release: bump version"
on:
workflow_dispatch:
inputs:
bump:
description: "Which part to increment (ignored when 'version' is set)"
type: choice
options:
- minor
- major
default: minor
version:
description: "Explicit MAJOR.MINOR to set, e.g. 4.36 (overrides 'bump')"
type: string
required: false
permissions:
contents: write
jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Compute new version
id: compute
env:
BUMP: ${{ inputs.bump }}
INPUT_VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
CONST=weed/util/version/constants.go
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
echo "current: ${MAJOR}.${MINOR}"
if [ -n "$INPUT_VERSION" ]; then
if ! [[ "$INPUT_VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
echo "::error::version must be MAJOR.MINOR, e.g. 5.01 (got '$INPUT_VERSION')"
exit 1
fi
# 10# forces base 10 so 08/09 are not parsed as octal.
MAJOR=$((10#${INPUT_VERSION%%.*}))
MINOR=$((10#${INPUT_VERSION##*.}))
if [ "$MINOR" -gt 99 ]; then
echo "::error::minor must be 0-99 (got $MINOR); it rolls into the next major at 99"
exit 1
fi
else
case "$BUMP" in
# Minor is a 2-digit field: 4.99 -> 5.00 -> 5.01.
major) MAJOR=$((MAJOR + 1)); MINOR=0 ;;
minor)
if [ "$MINOR" -ge 99 ]; then
MAJOR=$((MAJOR + 1)); MINOR=0
else
MINOR=$((MINOR + 1))
fi
;;
*) echo "::error::unknown bump '$BUMP'"; exit 1 ;;
esac
fi
# appVersion mirrors the Go VERSION_NUMBER (zero-padded minor);
# chart version is plain SemVer (no leading zeros).
APP_VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
CHART_VERSION="${MAJOR}.${MINOR}.0"
echo "new: app=${APP_VERSION} chart=${CHART_VERSION}"
{
echo "major=${MAJOR}"
echo "minor=${MINOR}"
echo "app_version=${APP_VERSION}"
echo "chart_version=${CHART_VERSION}"
} >> "$GITHUB_OUTPUT"
- name: Apply version to constants.go
env:
MAJOR: ${{ steps.compute.outputs.major }}
MINOR: ${{ steps.compute.outputs.minor }}
run: |
set -euo pipefail
CONST=weed/util/version/constants.go
sed -i -E "s/(MAJOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MAJOR}\2/" "$CONST"
sed -i -E "s/(MINOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MINOR}\2/" "$CONST"
grep -E 'MAJOR_VERSION|MINOR_VERSION' "$CONST"
- name: Apply version to Chart.yaml
env:
APP_VERSION: ${{ steps.compute.outputs.app_version }}
CHART_VERSION: ${{ steps.compute.outputs.chart_version }}
run: |
set -euo pipefail
CHART=k8s/charts/seaweedfs/Chart.yaml
sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "$CHART"
sed -i -E "s/^version:.*/version: ${CHART_VERSION}/" "$CHART"
cat "$CHART"
- name: Commit and push
env:
APP_VERSION: ${{ steps.compute.outputs.app_version }}
run: |
set -euo pipefail
if git diff --quiet; then
echo "No version change to commit."
exit 0
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add weed/util/version/constants.go k8s/charts/seaweedfs/Chart.yaml
git commit -m "${APP_VERSION}"
git push
@@ -1,244 +0,0 @@
name: "Rust Volume Server Tests"
on:
pull_request:
branches: [ master ]
paths:
- 'seaweed-volume/**'
- 'test/volume_server/**'
- 'weed/pb/volume_server.proto'
- 'weed/pb/volume_server_pb/**'
- '.github/workflows/rust-volume-server-tests.yml'
push:
branches: [ master, main ]
paths:
- 'seaweed-volume/**'
- 'test/volume_server/**'
- 'weed/pb/volume_server.proto'
- 'weed/pb/volume_server_pb/**'
- '.github/workflows/rust-volume-server-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
rust-unit-tests:
name: Rust Unit Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Build Rust volume server
run: cd seaweed-volume && cargo build --release
- name: Run Rust unit tests
run: cd seaweed-volume && cargo test
rust-integration-tests:
name: Rust Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Build Go weed binary
run: |
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
- name: Run integration tests
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
run: |
echo "Running Rust volume server integration tests..."
go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
- name: Collect logs on failure
if: failure()
run: |
mkdir -p /tmp/rust-volume-server-it-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: rust-volume-server-integration-test-logs
path: /tmp/rust-volume-server-it-logs/
if-no-files-found: warn
retention-days: 7
- name: Test summary
if: always()
run: |
echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
rust-volume-go-tests:
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
runs-on: ubuntu-22.04
timeout-minutes: 45
env:
# Keep in step with the length of matrix.shard below.
SHARD_COUNT: 3
strategy:
fail-fast: false
matrix:
test-type: [grpc, http]
shard: [1, 2, 3]
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Build Go weed binary
run: |
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
# Dealing the listed tests out one by one keeps the shards even. Bucketing
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
# the 114 grpc tests and ran nearly twice as long as the other two shards.
- name: Select this shard's tests
env:
TEST_TYPE: ${{ matrix.test-type }}
SHARD: ${{ matrix.shard }}
run: |
tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
# An empty list would make -run match nothing and the shard pass vacuously.
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
- name: Run volume server integration tests with Rust volume
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
VOLUME_SERVER_IMPL: rust
run: |
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
- name: Collect logs on failure
if: failure()
run: |
mkdir -p /tmp/rust-volume-go-test-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
path: /tmp/rust-volume-go-test-logs/
if-no-files-found: warn
retention-days: 7
- name: Test summary
if: always()
run: |
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
-166
View File
@@ -1,166 +0,0 @@
name: "rust: build dev volume server binaries"
on:
push:
branches: [ master ]
paths:
- 'seaweed-volume/**'
- '.github/workflows/rust_binaries_dev.yml'
permissions:
contents: read
jobs:
cleanup:
permissions:
contents: write
runs-on: ubuntu-latest
steps:
- name: Delete old Rust volume dev assets
uses: mknejp/delete-release-assets@v1
continue-on-error: true
with:
token: ${{ github.token }}
tag: dev
fail-if-no-assets: false
assets: |
weed-volume-*
build-rust-volume-dev-linux:
permissions:
contents: write
needs: cleanup
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
asset_suffix: linux-amd64
steps:
- uses: actions/checkout@v7
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-dev-${{ matrix.target }}-
- name: Set BUILD_TIME
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: cd seaweed-volume && cargo build --release
- name: Package large disk binary
run: |
cp seaweed-volume/target/release/weed-volume weed-volume-large-disk
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
rm weed-volume-large-disk
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: cd seaweed-volume && cargo build --release --no-default-features
- name: Package normal binary
run: |
cp seaweed-volume/target/release/weed-volume weed-volume-normal
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
rm weed-volume-normal
- name: Upload dev release assets
uses: softprops/action-gh-release@v3
with:
tag_name: dev
prerelease: true
files: |
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build-rust-volume-dev-darwin:
permissions:
contents: write
needs: build-rust-volume-dev-linux
runs-on: macos-latest
strategy:
matrix:
include:
- target: aarch64-apple-darwin
asset_suffix: darwin-arm64
- target: x86_64-apple-darwin
asset_suffix: darwin-amd64
steps:
- uses: actions/checkout@v7
- name: Install protobuf compiler
run: brew install protobuf
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-dev-${{ matrix.target }}-
- name: Set BUILD_TIME
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }}
- name: Package large disk binary
run: |
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
rm weed-volume-large-disk
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features
- name: Package normal binary
run: |
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
rm weed-volume-normal
- name: Upload dev release assets
uses: softprops/action-gh-release@v3
with:
tag_name: dev
prerelease: true
files: |
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-284
View File
@@ -1,284 +0,0 @@
name: "rust: build versioned volume server binaries"
on:
push:
tags:
- '*'
workflow_dispatch:
permissions:
contents: read
jobs:
build-rust-volume-linux:
permissions:
contents: write
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
asset_suffix: linux_amd64
- target: aarch64-unknown-linux-gnu
asset_suffix: linux_arm64
cross: true
steps:
- uses: actions/checkout@v7
- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install cross-compilation tools
if: matrix.cross
run: |
sudo dpkg --add-architecture arm64
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-release-${{ matrix.target }}-
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
- name: Package binaries
run: |
# Large disk (default, 5bytes feature)
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
rm weed-volume-large-disk
# Normal volume size
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
rm weed-volume-normal
- name: Generate md5 checksums
run: |
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
md5sum "$f" > "$f.md5"
done
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
with:
files: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
weed-volume_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.asset_suffix }}
path: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
weed-volume_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
build-rust-volume-darwin:
permissions:
contents: write
runs-on: macos-latest
strategy:
matrix:
include:
- target: x86_64-apple-darwin
asset_suffix: darwin_amd64
- target: aarch64-apple-darwin
asset_suffix: darwin_arm64
steps:
- uses: actions/checkout@v7
- name: Install protobuf compiler
run: brew install protobuf
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-release-${{ matrix.target }}-
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
- name: Package binaries
run: |
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
rm weed-volume-large-disk
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
rm weed-volume-normal
- name: Generate md5 checksums
run: |
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
md5 -r "$f" > "$f.md5"
done
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
with:
files: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
weed-volume_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-${{ matrix.asset_suffix }}
path: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
weed-volume_${{ matrix.asset_suffix }}.tar.gz
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
build-rust-volume-windows:
permissions:
contents: write
runs-on: windows-latest
steps:
- uses: actions/checkout@v7
- name: Install protobuf compiler
run: choco install protoc -y
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-release-windows-
- name: Build Rust volume server (large disk)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --target-dir target/large-disk
- name: Build Rust volume server (normal)
env:
SEAWEEDFS_COMMIT: ${{ github.sha }}
run: |
cd seaweed-volume
cargo build --release --no-default-features --target-dir target/normal
- name: Package binaries
shell: bash
run: |
cp seaweed-volume/target/large-disk/release/weed-volume.exe weed-volume-large-disk.exe
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
rm weed-volume-large-disk.exe
cp seaweed-volume/target/normal/release/weed-volume.exe weed-volume-normal.exe
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
rm weed-volume-normal.exe
- name: Generate md5 checksums
shell: bash
run: |
for f in weed-volume_large_disk_windows_amd64.zip weed-volume_windows_amd64.zip; do
md5sum "$f" > "$f.md5"
done
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
with:
files: |
weed-volume_large_disk_windows_amd64.zip
weed-volume_large_disk_windows_amd64.zip.md5
weed-volume_windows_amd64.zip
weed-volume_windows_amd64.zip.md5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-volume-windows_amd64
path: |
weed-volume_large_disk_windows_amd64.zip
weed-volume_large_disk_windows_amd64.zip.md5
weed-volume_windows_amd64.zip
weed-volume_windows_amd64.zip.md5
-129
View File
@@ -1,129 +0,0 @@
name: "S3 ETag and ACL Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/etag*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/etag/**'
- 'test/s3/acl/**'
- '.github/workflows/s3-etag-acl-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/etag*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/etag/**'
- 'test/s3/acl/**'
- '.github/workflows/s3-etag-acl-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-etag-acl-tests:
name: S3 ETag + ACL Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Start weed mini (S3 on :8333)
run: |
mkdir -p /tmp/seaweedfs-etag-acl
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
{
"identities": [
{
"name": "admin",
"credentials": [
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
],
"actions": ["Admin", "Read", "Write"]
}
]
}
JSON
AWS_ACCESS_KEY_ID=some_access_key1 \
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
weed mini \
-dir=/tmp/seaweedfs-etag-acl \
-s3.port=8333 \
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
-ip=127.0.0.1 \
> /tmp/weed-mini.log 2>&1 &
echo $! > /tmp/weed-mini.pid
# Wait for the S3 endpoint to come up (returns 403 unauth before any
# request is signed; that's fine — it means the server is listening).
for i in $(seq 1 30); do
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
echo "weed mini is ready"
exit 0
fi
sleep 1
done
echo "weed mini failed to start within 30s"
tail -50 /tmp/weed-mini.log
exit 1
- name: Run ETag tests
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
# SDK for Java v2 rejects the latter on the PutObject path.
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/etag/...
- name: Run ACL versioning tests
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
# PutObjectAcl with and without versionId, modifying ACLs on different
# versions independently.
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/acl/...
- name: Stop weed mini
if: always()
run: |
if [ -f /tmp/weed-mini.pid ]; then
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
fi
- name: Show server log on failure
if: failure()
run: |
echo "=== weed mini log (last 200 lines) ==="
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
- name: Archive log
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-etag-acl-server-log
path: /tmp/weed-mini.log
retention-days: 3
@@ -2,16 +2,7 @@ name: "S3 Authenticated Integration Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/iam/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/normal/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-example-integration-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-integration-tests
cancel-in-progress: true
@@ -27,10 +18,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -48,22 +39,6 @@ jobs:
echo "=== Running S3 Integration Tests ==="
go test -v -timeout=60s -run TestS3Integration ./...
- name: Run S3 DeleteBucketNotEmpty Tests
timeout-minutes: 15
working-directory: test/s3/normal
run: |
set -x
echo "=== Running S3 DeleteBucketNotEmpty Tests ==="
go test -v -timeout=60s -run TestS3DeleteBucketNotEmpty ./...
- name: Run S3 Empty Directory Marker Tests
timeout-minutes: 15
working-directory: test/s3/normal
run: |
set -x
echo "=== Running S3 Empty Directory Marker Tests ==="
go test -v -timeout=180s -run TestS3ListObjectsEmptyDirectoryMarkers ./...
- name: Run IAM Integration Tests
timeout-minutes: 15
working-directory: test/s3/normal
@@ -74,7 +49,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: integration-test-logs
path: test/s3/normal/*.log
+4 -12
View File
@@ -2,15 +2,7 @@ name: "S3 Filer Group Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/filer_group/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-filer-group-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-filer-group-tests
cancel-in-progress: true
@@ -30,10 +22,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -85,7 +77,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-filer-group-test-logs
path: test/s3/filer_group/weed-test*.log
+28 -164
View File
@@ -2,15 +2,7 @@ name: "S3 Go Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-go-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-go-tests
cancel-in-progress: true
@@ -33,10 +25,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -84,7 +76,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-test-logs-${{ matrix.test-type }}
path: test/s3/versioning/weed-test*.log
@@ -97,10 +89,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -132,7 +124,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-compatibility-logs
path: test/s3/versioning/weed-test*.log
@@ -145,10 +137,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -180,7 +172,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-cors-compatibility-logs
path: test/s3/cors/weed-test*.log
@@ -196,10 +188,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -247,138 +239,12 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-retention-test-logs-${{ matrix.test-type }}
path: test/s3/retention/weed-test*.log
retention-days: 3
s3-lifecycle-tests:
name: S3 Lifecycle Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
# One job per test so each gets a fresh `weed mini` server, avoiding
# the cross-test volume-pool exhaustion that surfaced when several
# TTL-pinned bucket collections piled up in a single run.
test:
- TestLifecycleAbortIncompleteMultipartUpload
- TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort
- TestLifecycleBootstrapWalkOnExistingObjects
- TestLifecycleConfigUpdateBetweenSweeps
- TestLifecycleDeleteBucketLifecycleStopsDispatching
- TestLifecycleDisabledRuleSkipsObject
- TestLifecycleEmptyBucketSweepIsNoOp
- TestLifecycleExpirationDateInThePast
- TestLifecycleExpirationFiresOnBackdatedObject
- TestLifecycleExpiredDeleteMarkerCleanup
- TestLifecycleMultipleBucketsInOneSweep
- TestLifecycleMultipleRulesInOneBucket
- TestLifecycleNewerNoncurrentVersions
- TestLifecycleNoncurrentVersionExpiration
- TestLifecycleSizeFilterGreaterThan
- TestLifecycleSkipsObjectLockedObjects
- TestLifecycleSuspendedVersioningExpiration
- TestLifecycleTagFilter
- TestLifecycleVersionedBucketCreatesDeleteMarker
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run ${{ matrix.test }}
timeout-minutes: 8
working-directory: test/s3/lifecycle
run: |
set -x
make test-with-server TEST_PATTERN='^${{ matrix.test }}$$'
- name: Show server logs on failure
if: failure()
working-directory: test/s3/lifecycle
run: |
if [ -f weed-test.log ]; then
echo "=== Last 200 lines of server logs ==="
tail -200 weed-test.log
fi
ps aux | grep -E "(weed|test)" || true
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-lifecycle-test-logs-${{ matrix.test }}
path: test/s3/lifecycle/weed-test*.log
retention-days: 3
s3-checksum-tests:
name: S3 Checksum Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Checksum Tests
timeout-minutes: 16
working-directory: test/s3/checksum
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Tests ==="
make test-with-server
- name: Show server logs on failure
if: failure()
working-directory: test/s3/checksum
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-checksum-test-logs
path: test/s3/checksum/weed-test*.log
retention-days: 3
s3-cors-tests:
name: S3 CORS Tests
runs-on: ubuntu-22.04
@@ -389,10 +255,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -440,7 +306,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-cors-test-logs-${{ matrix.test-type }}
path: test/s3/cors/weed-test*.log
@@ -453,10 +319,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -489,7 +355,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-retention-worm-logs
path: test/s3/retention/weed-test*.log
@@ -502,10 +368,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -556,7 +422,7 @@ jobs:
- name: Upload stress test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-stress-logs
path: test/s3/versioning/weed-test*.log
@@ -570,10 +436,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -595,8 +461,6 @@ jobs:
export S3_ENDPOINT="http://localhost:8006"
export S3_ACCESS_KEY="0555b35654ad1656d804"
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
# Run the specific test that is equivalent to AWS S3 tagging behavior
make test-with-server || {
@@ -612,7 +476,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-tagging-test-logs
path: test/s3/tagging/weed-test*.log
@@ -625,10 +489,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -665,7 +529,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-remote-cache-test-logs
path: |
+14 -40
View File
@@ -5,8 +5,6 @@ on:
paths:
- 'weed/iam/**'
- 'weed/s3api/**'
- 'weed/credential/**'
- 'weed/pb/**'
- 'test/s3/iam/**'
- '.github/workflows/s3-iam-tests.yml'
push:
@@ -14,8 +12,6 @@ on:
paths:
- 'weed/iam/**'
- 'weed/s3api/**'
- 'weed/credential/**'
- 'weed/pb/**'
- 'test/s3/iam/**'
- '.github/workflows/s3-iam-tests.yml'
@@ -39,10 +35,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -69,7 +65,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: iam-unit-test-results
path: |
@@ -84,14 +80,14 @@ jobs:
timeout-minutes: 25
strategy:
matrix:
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
test-type: ["basic", "advanced", "policy-enforcement"]
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -121,7 +117,7 @@ jobs:
"basic")
echo "Running basic IAM functionality tests..."
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./...
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAM" ./...
;;
"advanced")
echo "Running advanced IAM feature tests..."
@@ -133,28 +129,6 @@ jobs:
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
;;
"group")
echo "Running IAM group management tests..."
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestIAMGroup" ./...
;;
"sts")
echo "Running STS and service account tests..."
make clean setup start-services wait-for-services
# SigV4-signed STS calls need admin credentials matching test_config.json.
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
export STS_TEST_ACCESS_KEY=test-access-key
export STS_TEST_SECRET_KEY=test-secret-key
# The use_service_account_credentials subtest is excluded because
# newly-created service-account access keys are not currently
# persisted to the filer after CreateServiceAccount — a
# pre-existing sync issue tracked separately from the
# GetFederationToken routing fix this PR addresses.
go test -v -timeout 15m \
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
./...
;;
*)
echo "Unknown test type: ${{ matrix.test-type }}"
exit 1
@@ -188,7 +162,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-integration-logs-${{ matrix.test-type }}
path: test/s3/iam/weed-*.log
@@ -202,10 +176,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -248,7 +222,7 @@ jobs:
- name: Upload distributed test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-distributed-logs
path: test/s3/iam/weed-*.log
@@ -262,10 +236,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -300,7 +274,7 @@ jobs:
- name: Upload performance test results
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-performance-results
path: |
+4 -4
View File
@@ -35,10 +35,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -97,7 +97,7 @@ jobs:
# Verify service accessibility
echo "=== Verifying Service Accessibility ==="
curl -f http://localhost:8080/realms/master
curl -s http://localhost:8333/healthz
curl -s http://localhost:8333
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
# Run Keycloak-specific tests
@@ -152,7 +152,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-keycloak-test-logs
path: |
@@ -1,148 +0,0 @@
name: "S3 Mutation Regression Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'test/s3/delete/**'
- 'test/s3/distributed_lock/**'
- 'test/s3/versioning/**'
- 'test/volume_server/framework/**'
- 'docker/compose/s3.json'
- '.github/workflows/s3-mutation-regression-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-mutation-regression-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-versioning-regressions:
name: S3 Versioning Regression Tests
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Run S3 versioning regression tests
timeout-minutes: 20
working-directory: test/s3/versioning
run: |
set -x
# Run every versioning test, so a regression test lands covered instead
# of waiting for someone to remember this file. Name a test in EXCLUDE,
# with the reason, to keep it out.
#
# TestVersioningPagination*: opt-in stress tests that build 1500+
# versions. They self-skip without ENABLE_STRESS_TESTS and have their
# own make target, so this gate should not carry them.
EXCLUDE='TestVersioningPagination.*'
tests=$(go test . -list '.*' | grep '^Test' | sort -u)
# An empty list would make -run match nothing and pass this job vacuously.
[ -n "$tests" ] || { echo "listed no versioning tests"; exit 1; }
selected=$(echo "$tests" | grep -vE "^($EXCLUDE)$")
[ -n "$selected" ] || { echo "EXCLUDE matched every test"; exit 1; }
echo "running $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) versioning tests"
# make swallows a lone trailing $, taking the anchor with it, so escape it.
make test-with-server TEST_PATTERN="^($(echo "$selected" | paste -sd'|' -))"'$$'
- name: Show server logs on failure
if: failure()
working-directory: test/s3/versioning
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
tail -100 weed-test.log
fi
- name: Upload versioning logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-versioning-regression-logs
path: test/s3/versioning/weed-test*.log
retention-days: 3
s3-delete-regressions:
name: S3 Delete Regression Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Run S3 delete regression tests
timeout-minutes: 15
working-directory: test/s3/delete
run: |
set -x
make test-with-server
- name: Show server logs on failure
if: failure()
working-directory: test/s3/delete
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
tail -100 weed-test.log
fi
- name: Upload delete logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-delete-regression-logs
path: test/s3/delete/weed-test*.log
retention-days: 3
s3-distributed-lock-regressions:
name: S3 Distributed Lock Regression Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS
run: |
go build -o weed/weed -buildvcs=false ./weed
- name: Run distributed lock regressions
timeout-minutes: 25
env:
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: |
set -x
mkdir -p "$TMPDIR"
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
- name: Upload distributed lock logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-distributed-lock-regression-logs
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
retention-days: 3
+8 -8
View File
@@ -36,16 +36,16 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
@@ -121,7 +121,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: test-logs-python-${{ matrix.python-version }}
path: |
@@ -143,12 +143,12 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
- name: Run Go unit tests
+14 -14
View File
@@ -43,10 +43,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -70,7 +70,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: policy-unit-test-results
path: |
@@ -86,10 +86,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -178,7 +178,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-policy-variables-test-logs
path: /tmp/weed_policy_test_server.log
@@ -195,10 +195,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -299,7 +299,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
@@ -313,10 +313,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -386,7 +386,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: trusted-proxy-test-logs
path: /tmp/weed_proxy_test.log
@@ -400,10 +400,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -1,143 +0,0 @@
name: "S3 Proxy Signature Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/server/**'
- 'test/s3/proxy_signature/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-proxy-signature-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/server/**'
- 'test/s3/proxy_signature/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-proxy-signature-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
proxy-signature-tests:
name: S3 Proxy Signature Verification Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Build SeaweedFS binary for Linux
run: |
set -x
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -v -o test/s3/proxy_signature/weed ./weed
- name: Run S3 Proxy Signature Tests
timeout-minutes: 10
working-directory: test/s3/proxy_signature
run: |
set -x
echo "Starting Docker Compose services..."
docker compose up -d --build
# Check if containers are running
echo "Checking container status..."
docker compose ps
# Wait for services to be ready
echo "Waiting for nginx proxy to be ready..."
PROXY_READY=0
for i in $(seq 1 30); do
if curl -s http://localhost:9000/ > /dev/null 2>&1; then
echo "Proxy is ready"
PROXY_READY=1
break
fi
echo "Waiting for proxy... ($i/30)"
sleep 1
done
if [ $PROXY_READY -eq 0 ]; then
echo "ERROR: Proxy failed to become ready after 30 seconds"
echo "Docker compose logs:"
docker compose logs --no-color || true
exit 1
fi
# Wait for SeaweedFS to be ready
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
S3_READY=0
for i in $(seq 1 30); do
# Check logs first for the readiness line. weed mini's progress
# board prints " S3 ready (Xs)"; older builds and the
# standalone S3 binary log "S3 (gateway|service) ... ready".
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then
echo "SeaweedFS S3 gateway is ready"
S3_READY=1
break
fi
# Fallback: check headers via proxy (which is already ready)
if curl -s -I http://localhost:9000/ | grep -qi "SeaweedFS"; then
echo "SeaweedFS S3 gateway is responding via proxy"
S3_READY=1
break
fi
echo "Waiting for S3 gateway... ($i/30)"
sleep 1
done
if [ $S3_READY -eq 0 ]; then
echo "ERROR: SeaweedFS S3 gateway failed to become ready after 30 seconds"
echo "Latest seaweedfs logs:"
docker compose logs --no-color --tail 20 seaweedfs || true
exit 1
fi
# Run the test script inside AWS CLI container
echo "Running test script..."
docker run --rm --network host \
--entrypoint bash \
amazon/aws-cli:latest \
-c "$(cat test.sh)"
TEST_RESULT=$?
# Cleanup
docker compose down
exit $TEST_RESULT
- name: Cleanup on failure
if: failure()
working-directory: test/s3/proxy_signature
run: |
echo "Cleaning up Docker containers..."
ls -al weed || true
ldd weed || true
echo "Docker compose logs:"
docker compose logs --no-color || true
echo "Container status before cleanup:"
docker ps -a
echo "Stopping services..."
docker compose down || true
@@ -1,110 +0,0 @@
name: "S3 SDK V2 Route Disambiguation Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'test/s3/sdk_v2_routing/**'
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'test/s3/sdk_v2_routing/**'
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-sdk-v2-routing-tests:
name: S3 SDK V2 Routing Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Start weed mini (S3 on :8333)
# Pins the regression for issue #9559: AWS SDK V2 / Hadoop s3a
# listing a bucket literally named "buckets" must get an XML
# ListObjectsV2 response, not the JSON ListTableBuckets body
# served by the S3 Tables REST endpoint on the same path.
run: |
mkdir -p /tmp/seaweedfs-sdk-v2-routing
cat > /tmp/seaweedfs-sdk-v2-routing-s3.json <<'JSON'
{
"identities": [
{
"name": "admin",
"credentials": [
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
],
"actions": ["Admin", "Read", "Write"]
}
]
}
JSON
AWS_ACCESS_KEY_ID=some_access_key1 \
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
weed mini \
-dir=/tmp/seaweedfs-sdk-v2-routing \
-s3.port=8333 \
-s3.config=/tmp/seaweedfs-sdk-v2-routing-s3.json \
-ip=127.0.0.1 \
> /tmp/weed-mini.log 2>&1 &
echo $! > /tmp/weed-mini.pid
for i in $(seq 1 30); do
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
echo "weed mini is ready"
exit 0
fi
sleep 1
done
echo "weed mini failed to start within 30s"
tail -50 /tmp/weed-mini.log
exit 1
- name: Run SDK V2 routing tests
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/sdk_v2_routing/...
- name: Stop weed mini
if: always()
run: |
if [ -f /tmp/weed-mini.pid ]; then
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
fi
- name: Show server log on failure
if: failure()
run: |
echo "=== weed mini log (last 200 lines) ==="
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
- name: Archive log
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-sdk-v2-routing-server-log
path: /tmp/weed-mini.log
retention-days: 3
+6 -11
View File
@@ -4,7 +4,6 @@ on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'test/s3/spark/**'
- 'test/s3tables/testutil/**'
- '.github/workflows/s3-spark-tests.yml'
@@ -25,27 +24,23 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v3
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Pre-pull Spark image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/spark:3.5.1
run: docker pull apache/spark:3.5.8
- name: Run S3 Spark integration tests
working-directory: test/s3/spark
@@ -78,7 +73,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-spark-test-logs
path: test/s3/spark/test-output.log
+23 -27
View File
@@ -44,10 +44,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -73,12 +73,8 @@ jobs:
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
else
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
# same change that fixed them.
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
fi
- name: Show server logs on failure
@@ -99,7 +95,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-test-logs-${{ matrix.test-type }}
path: test/s3/sse/weed-test*.log
@@ -112,10 +108,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -147,7 +143,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-compatibility-logs
path: test/s3/sse/weed-test*.log
@@ -160,10 +156,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -196,7 +192,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-metadata-persistence-logs
path: test/s3/sse/weed-test*.log
@@ -209,10 +205,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -245,7 +241,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-copy-operations-logs
path: test/s3/sse/weed-test*.log
@@ -258,10 +254,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -294,7 +290,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-multipart-logs
path: test/s3/sse/weed-test*.log
@@ -309,10 +305,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -344,7 +340,7 @@ jobs:
- name: Upload performance test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-performance-logs
path: test/s3/sse/weed-test*.log
@@ -357,10 +353,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -393,7 +389,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-volume-encryption-logs
path: /tmp/seaweedfs-sse-*.log
+54 -380
View File
@@ -2,14 +2,10 @@ name: "S3 Tables Integration Tests"
on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3tables/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3-tables-tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3-tables-tests
cancel-in-progress: true
permissions:
contents: read
@@ -23,10 +19,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -70,7 +66,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-tables-test-logs
path: test/s3tables/table-buckets/test-output.log
@@ -83,19 +79,14 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Run go mod tidy
run: go mod tidy
@@ -131,7 +122,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: iceberg-catalog-test-logs
path: test/s3tables/catalog/test-output.log
@@ -144,23 +135,19 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v3
- name: Pre-pull Trino image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull trinodb/trino:479
run: docker pull trinodb/trino:479
- name: Run go mod tidy
run: go mod tidy
@@ -201,226 +188,12 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: trino-iceberg-catalog-test-logs
path: test/s3tables/catalog_trino/test-output.log
retention-days: 3
dremio-iceberg-catalog-tests:
name: Dremio Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Pre-pull images
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull dremio/dremio-oss:25.2.0
pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Dremio Iceberg Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/catalog_dremio
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Dremio Iceberg Catalog Tests ==="
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Dremio Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_dremio
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|dremio)" || true
echo "=== Dremio containers ==="
docker ps -a --filter "name=seaweed-dremio" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: dremio-iceberg-catalog-test-logs
path: test/s3tables/catalog_dremio/test-output.log
retention-days: 3
doris-iceberg-catalog-tests:
name: Doris Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 35
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Pre-pull images
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/doris:doris-all-in-one-2.1.0
pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Doris Iceberg Catalog Integration Tests
timeout-minutes: 30
working-directory: test/s3tables/catalog_doris
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Doris Iceberg Catalog Tests ==="
go test -v -timeout 25m . 2>&1 | tee test-output.log || {
echo "Doris Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_doris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|doris)" || true
echo "=== Doris containers ==="
docker ps -a --filter "name=seaweed-doris" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: doris-iceberg-catalog-test-logs
path: test/s3tables/catalog_doris/test-output.log
retention-days: 3
polaris-integration-tests:
name: Polaris Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Pre-pull Polaris image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/polaris:latest
- name: Run Polaris Integration Tests
timeout-minutes: 25
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Polaris Tests ==="
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
echo "Polaris integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/polaris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: polaris-test-logs
path: test/s3tables/polaris/test-output.log
retention-days: 3
spark-iceberg-catalog-tests:
name: Spark Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
@@ -428,23 +201,19 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v3
- name: Pre-pull Spark image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/spark:3.5.1
run: docker pull apache/spark:3.5.1
- name: Run go mod tidy
run: go mod tidy
@@ -485,7 +254,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: spark-iceberg-catalog-test-logs
path: test/s3tables/catalog_spark/test-output.log
@@ -498,24 +267,21 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v3
- name: Pre-pull RisingWave image
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull risingwavelabs/risingwave:v2.5.0
pull postgres:16-alpine
docker pull risingwavelabs/risingwave:v2.5.0
docker pull postgres:16-alpine
- name: Run go mod tidy
run: go mod tidy
@@ -556,7 +322,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: risingwave-catalog-test-logs
path: test/s3tables/catalog_risingwave/test-output.log
@@ -569,47 +335,19 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Set up Docker
uses: docker/setup-buildx-action@v3
- name: Week stamp for image cache key
id: week
run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
- name: Restore python:3 image cache
id: python-image
uses: actions/cache@v6
with:
path: /tmp/python3-image.tar
key: python3-image-${{ steps.week.outputs.week }}
restore-keys: |
python3-image-
- name: Load or pull python:3
run: |
if [ "${{ steps.python-image.outputs.cache-hit }}" = "true" ]; then
docker load -i /tmp/python3-image.tar
exit 0
fi
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
if pull python:3; then
docker save -o /tmp/python3-image.tar python:3
elif [ -f /tmp/python3-image.tar ]; then
# Docker Hub unreachable; fall back to last week's cached image
docker load -i /tmp/python3-image.tar
else
exit 1
fi
- name: Pre-pull Python image
run: docker pull python:3
- name: Run go mod tidy
run: go mod tidy
@@ -650,7 +388,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: sts-integration-test-logs
path: test/s3tables/sts_integration/test-output.log
@@ -663,14 +401,23 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v3
- name: Pre-pull Python image
run: docker pull python:3
- name: Pre-pull LocalStack image (if needed)
run: docker pull localstack/localstack:latest || true
- name: Run go mod tidy
run: go mod tidy
@@ -710,85 +457,12 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: lakekeeper-integration-test-logs
path: test/s3tables/lakekeeper/test-output.log
retention-days: 3
unity-catalog-integration-tests:
name: Unity Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
id: go
- name: Configure Docker Hub mirror
run: |
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
- name: Pre-pull images
run: |
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull unitycatalog/unitycatalog:v0.4.0
pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Unity Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/unity_catalog
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Unity Catalog Tests ==="
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Unity Catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/unity_catalog
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
echo "=== Unity Catalog containers ==="
docker ps -a --filter "name=seaweed-unity-catalog" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: unity-catalog-integration-test-logs
path: test/s3tables/unity_catalog/test-output.log
retention-days: 3
s3-tables-build-verification:
name: S3 Tables Build Verification
runs-on: ubuntu-22.04
@@ -796,10 +470,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -861,10 +535,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -912,10 +586,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
+29 -126
View File
@@ -3,26 +3,8 @@ name: "Ceph S3 tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'weed/iam/**'
- 'test/s3/compatibility/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'weed/iam/**'
- 'test/s3/compatibility/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/s3tests.yml'
concurrency:
group: ${{ github.head_ref }}/s3tests
@@ -38,16 +20,16 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: '3.9'
@@ -55,15 +37,13 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Fix S3 tests bucket creation conflicts
run: |
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
python3 test/s3/fix_s3_tests_bucket_conflicts.py
env:
S3_TESTS_PATH: s3-tests
@@ -150,49 +130,7 @@ jobs:
done
echo "✅ S3 server is responding, starting tests..."
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
# something driving deletions. The s3tests build tag rescales one
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
# the upload's mtime; -dispatch / -checkpoint defaults are already
# tightened under the same build tag.
LC_LOG=/tmp/lifecycle-worker.log
# -debug routes glog to stderr so the bootstrap walker's progress
# shows up in $LC_LOG; without it weed shell silences glog.
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
| weed shell -debug -master=localhost:9333 \
> "$LC_LOG" 2>&1 &
lc_pid=$!
# Aliveness check: a bad shell command exits in <1s and the suite
# would otherwise just timeout the expiration tests with no signal.
sleep 2
if ! kill -0 "$lc_pid" 2>/dev/null; then
echo "lifecycle worker died on startup"
tail -50 "$LC_LOG" 2>/dev/null || true
exit 1
fi
echo "lifecycle worker pid=$lc_pid"
# bash -e exits the step on the first tox failure, so move teardown
# into a trap to guarantee the worker log + data dir reach the runner.
cleanup() {
status=$?
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
# bash fallback if it ignores TERM. Reading the log AFTER the
# graceful-stop window catches the bootstrap walker's progress.
kill -TERM "$lc_pid" 2>/dev/null || true
kill -TERM "$pid" 2>/dev/null || true
sleep 1
if [ "$status" -ne 0 ]; then
echo "=== lifecycle worker log (tail) ==="
tail -200 "$LC_LOG" 2>/dev/null || true
fi
kill -9 "$lc_pid" 2>/dev/null || true
kill -9 "$pid" 2>/dev/null || true
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
}
trap cleanup EXIT
tox -- \
s3tests/functional/test_s3.py::test_bucket_list_empty \
s3tests/functional/test_s3.py::test_bucket_list_distinct \
@@ -372,8 +310,11 @@ jobs:
s3tests/functional/test_s3.py::test_lifecycle_get \
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
s3tests/functional/test_s3.py::test_lifecycle_expiration \
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
# cleanup() trap handles worker/server kill + data dir wipe.
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
versioning-tests:
name: S3 Versioning & Object Lock tests
@@ -381,16 +322,16 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: '3.9'
@@ -398,15 +339,13 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Fix S3 tests bucket creation conflicts
run: |
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
python3 test/s3/fix_s3_tests_bucket_conflicts.py
env:
S3_TESTS_PATH: s3-tests
@@ -551,16 +490,16 @@ jobs:
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: '3.9'
@@ -568,8 +507,6 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
@@ -676,10 +613,10 @@ jobs:
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -694,12 +631,10 @@ jobs:
export WEED_DATA_DIR="/tmp/seaweedfs-copy-test-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
set -x
# Each test bucket is its own collection and grows 7 volumes; the suite runs
# faster than the heartbeat that returns slots from the deleted collections.
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=300 -volume.preStopSeconds=1 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
pid=$!
@@ -767,7 +702,7 @@ jobs:
sleep 2
done
MASTER_ENDPOINT="http://127.0.0.1:9336" go test -v
go test -v
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
@@ -778,16 +713,16 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v7
uses: actions/setup-python@v6
with:
python-version: '3.9'
@@ -795,8 +730,6 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
@@ -1017,39 +950,6 @@ jobs:
sleep 2
done
# Spawn the lifecycle worker (see basic-tests block for context).
LC_LOG=/tmp/lifecycle-worker-sql.log
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
| weed shell -debug -master=localhost:9337 \
> "$LC_LOG" 2>&1 &
lc_pid=$!
sleep 2
if ! kill -0 "$lc_pid" 2>/dev/null; then
echo "lifecycle worker died on startup"
tail -50 "$LC_LOG" 2>/dev/null || true
exit 1
fi
echo "lifecycle worker pid=$lc_pid"
cleanup() {
status=$?
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
# bash fallback if it ignores TERM. Reading the log AFTER the
# graceful-stop window catches the bootstrap walker's progress.
kill -TERM "$lc_pid" 2>/dev/null || true
kill -TERM "$pid" 2>/dev/null || true
sleep 1
if [ "$status" -ne 0 ]; then
echo "=== lifecycle worker log (tail) ==="
tail -200 "$LC_LOG" 2>/dev/null || true
fi
kill -9 "$lc_pid" 2>/dev/null || true
kill -9 "$pid" 2>/dev/null || true
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
}
trap cleanup EXIT
tox -- \
s3tests/functional/test_s3.py::test_bucket_list_empty \
s3tests/functional/test_s3.py::test_bucket_list_distinct \
@@ -1229,7 +1129,10 @@ jobs:
s3tests/functional/test_s3.py::test_lifecycle_get \
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
s3tests/functional/test_s3.py::test_lifecycle_expiration \
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
# cleanup() trap handles worker/server kill + data dir wipe.
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
-120
View File
@@ -1,120 +0,0 @@
name: "Samba on FUSE Integration"
on:
push:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'weed/cluster/**'
- 'test/samba/**'
- '.github/workflows/samba-integration.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'weed/cluster/**'
- 'test/samba/**'
- '.github/workflows/samba-integration.yml'
workflow_dispatch:
concurrency:
group: samba-integration/${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
samba-integration:
name: samba-integration
runs-on: ubuntu-22.04
timeout-minutes: 45
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Start local Docker registry
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Build weed race binary
run: |
cd docker
make binary_race
- name: Build SeaweedFS e2e image
uses: docker/build-push-action@v7
with:
context: docker
file: docker/Dockerfile.e2e
tags: localhost:5000/chrislusf/seaweedfs:e2e
push: true
cache-from: type=gha,scope=samba-e2e
cache-to: type=gha,mode=max,scope=samba-e2e
- name: Tag e2e image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:e2e
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
- name: Build samba image
uses: docker/build-push-action@v7
with:
context: test/samba
build-contexts: |
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
tags: localhost:5000/chrislusf/seaweedfs:samba
push: true
cache-from: type=gha,scope=samba-harness
cache-to: type=gha,mode=max,scope=samba-harness
- name: Tag samba image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:samba
docker tag localhost:5000/chrislusf/seaweedfs:samba chrislusf/seaweedfs:samba
- name: Start SeaweedFS cluster and Samba
run: |
docker compose -f test/samba/docker-compose.yml up --wait
- name: Run Samba test battery
run: |
set -o pipefail
docker compose -f test/samba/docker-compose.yml exec -T samba \
/run_inside_container.sh 2>&1 | tee /tmp/samba-output.log
- name: Collect logs
if: always()
run: |
mkdir -p /tmp/samba-docker-logs
for svc in master volume filer samba; do
docker compose -f test/samba/docker-compose.yml logs "$svc" \
> "/tmp/samba-docker-logs/${svc}.log" 2>&1 || true
done
- name: Tear down
if: always()
run: |
docker compose -f test/samba/docker-compose.yml down -v
- name: Upload logs
if: always()
uses: actions/upload-artifact@v7
with:
name: samba-integration-results
path: |
/tmp/samba-output.log
/tmp/samba-docker-logs/
retention-days: 7
+5 -4
View File
@@ -24,6 +24,7 @@ permissions:
contents: read
env:
GO_VERSION: '1.24'
TEST_TIMEOUT: '15m'
jobs:
@@ -34,12 +35,12 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: ${{ env.GO_VERSION }}
- name: Install dependencies
run: |
@@ -31,7 +31,7 @@ jobs:
# SETUP & BUILD
# ========================================
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up JDK 11
uses: actions/setup-java@v5
@@ -41,9 +41,9 @@ jobs:
cache: maven
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: '1.24'
- name: Build SeaweedFS binary
run: |
@@ -125,7 +125,7 @@ jobs:
- name: Upload test results
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: spark-test-results
path: test/java/spark/target/surefire-reports/
@@ -133,7 +133,7 @@ jobs:
- name: Publish test report
if: always()
uses: dorny/test-reporter@v3
uses: dorny/test-reporter@v2
with:
name: Spark Test Results
path: test/java/spark/target/surefire-reports/*.xml
@@ -150,7 +150,7 @@ jobs:
- name: Cache Apache Spark
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
id: cache-spark
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: spark-3.5.0-bin-hadoop3
key: spark-3.5.0-hadoop3
@@ -1,46 +0,0 @@
name: Telemetry Integration Tests
on:
push:
branches: [ master ]
paths:
- 'telemetry/**'
- 'weed/telemetry/**'
- '.github/workflows/telemetry-integration.yml'
pull_request:
branches: [ master ]
paths:
- 'telemetry/**'
- 'weed/telemetry/**'
- '.github/workflows/telemetry-integration.yml'
permissions:
contents: read
jobs:
telemetry-integration-test:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
- name: Build telemetry server
run: cd telemetry/server && go build -o telemetry-server .
- name: Run telemetry integration test
run: go run telemetry/test/integration.go
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: telemetry-test-logs
path: telemetry-server-test.log
retention-days: 7
-80
View File
@@ -1,80 +0,0 @@
name: "terraform: validate and test modules"
on:
push:
branches: [ master ]
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
pull_request:
branches: [ master ]
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
permissions:
contents: read
jobs:
validate:
name: fmt, validate, plan-level tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up OpenTofu
uses: opentofu/setup-opentofu@v2
with:
tofu_version: 1.12.1
- name: fmt check
working-directory: terraform
run: tofu fmt -recursive -check -diff
- name: validate core
working-directory: terraform/modules/core
run: |
tofu init -backend=false -input=false
tofu validate
- name: validate security
working-directory: terraform/modules/security
run: |
tofu init -backend=false -input=false
tofu validate
- name: plan-level tests (core)
working-directory: terraform/modules/core
run: tofu test
- name: validate examples
run: |
set -e
for ex in terraform/examples/*/; do
echo "== validate $ex =="
tofu -chdir="$ex" init -backend=false -input=false
tofu -chdir="$ex" validate
done
smoke:
name: local cluster smoke test (real weed)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
- name: Build weed
run: go build -o "$RUNNER_TEMP/weed" ./weed
- name: Set up OpenTofu
uses: opentofu/setup-opentofu@v2
with:
tofu_version: 1.12.1
- name: Run local cluster harness
working-directory: terraform/test/local
run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
- name: Run local mTLS cluster harness
working-directory: terraform/test/local-secure
run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh
@@ -3,20 +3,8 @@ name: "test s3 over https using aws-cli"
on:
push:
branches: [master, test-https-s3-awscli]
paths:
- 'weed/s3api/**'
- 'weed/server/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/test-s3-over-https-using-awscli.yml'
pull_request:
branches: [master, test-https-s3-awscli]
paths:
- 'weed/s3api/**'
- 'weed/server/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/test-s3-over-https-using-awscli.yml'
env:
AWS_ACCESS_KEY_ID: some_access_key1
@@ -32,11 +20,11 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- uses: actions/setup-go@v7
- uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
- name: Build SeaweedFS
run: |
@@ -47,7 +35,7 @@ jobs:
set -e
mkdir -p /tmp/data
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
- name: Setup Caddy
run: |
@@ -66,7 +54,7 @@ jobs:
- name: Start Caddy
run: |
./caddy start
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
- name: Create Bucket
run: |
@@ -115,7 +103,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: seaweedfs-logs
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
-68
View File
@@ -1,68 +0,0 @@
name: "TLS Rotation Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'test/tls_rotation/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/tls-rotation-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'test/tls_rotation/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/tls-rotation-tests.yml'
permissions:
contents: read
jobs:
tls-rotation-tests:
name: TLS Rotation Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run TLS Rotation Integration Tests
working-directory: test/tls_rotation
run: |
go test -v -count=1 -timeout 5m
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting master logs from temp directories..."
mkdir -p /tmp/tls-rotation-test-logs
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
fi
done
echo "Collected logs:"
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: tls-rotation-test-logs
path: /tmp/tls-rotation-test-logs/
retention-days: 14
+11 -7
View File
@@ -3,12 +3,16 @@ name: "TUS Protocol Tests"
on:
pull_request:
paths:
- 'weed/server/**'
- 'weed/filer/**'
- 'weed/server/filer_server_tus*.go'
- 'weed/server/filer_server.go'
- 'test/tus/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/tus-tests.yml'
push:
branches: [ master, main ]
paths:
- 'weed/server/filer_server_tus*.go'
- 'weed/server/filer_server.go'
- 'test/tus/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/tus-tests
@@ -29,10 +33,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
@@ -102,7 +106,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: tus-test-logs
path: |
@@ -1,68 +0,0 @@
name: "Vacuum Integration Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/**'
- 'test/vacuum/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/vacuum-integration-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/**'
- 'test/vacuum/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/vacuum-integration-tests.yml'
permissions:
contents: read
jobs:
vacuum-integration-tests:
name: Vacuum Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v7
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run Vacuum Integration Tests
working-directory: test/vacuum
run: |
go test -v -timeout 10m
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting server logs from temp directories..."
mkdir -p /tmp/vacuum-test-logs
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
fi
done
echo "Collected logs:"
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: vacuum-integration-test-logs
path: /tmp/vacuum-test-logs/
retention-days: 14
@@ -28,9 +28,8 @@ permissions:
contents: read
env:
GO_VERSION: '1.24'
TEST_TIMEOUT: '30m'
# Keep in step with the length of matrix.shard below.
SHARD_COUNT: 3
jobs:
volume-server-integration-tests:
@@ -45,12 +44,12 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v7
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: ${{ env.GO_VERSION }}
- name: Build SeaweedFS binary
run: |
@@ -59,26 +58,28 @@ jobs:
chmod +x weed
./weed version
# Dealing the listed tests out one by one keeps the shards even. Bucketing
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
# the 114 grpc tests and ran nearly twice as long as the other two shards.
- name: Select this shard's tests
env:
TEST_TYPE: ${{ matrix.test-type }}
SHARD: ${{ matrix.shard }}
run: |
tests=$(go test ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
# An empty list would make -run match nothing and the shard pass vacuously.
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
- name: Run volume server integration tests
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: |
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
- name: Collect logs on failure
@@ -89,7 +90,7 @@ jobs:
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: volume-server-integration-test-logs
path: /tmp/volume-server-it-logs/
@@ -99,6 +100,23 @@ jobs:
- name: Test summary
if: always()
run: |
if [ "${{ matrix.test-type }}" == "grpc" ]; then
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-H]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[I-S]"
else
TEST_PATTERN="^Test[T-Z]"
fi
else
if [ "${{ matrix.shard }}" == "1" ]; then
TEST_PATTERN="^Test[A-G]"
elif [ "${{ matrix.shard }}" == "2" ]; then
TEST_PATTERN="^Test[H-R]"
else
TEST_PATTERN="^Test[S-Z]"
fi
fi
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
-3
View File
@@ -2,7 +2,6 @@
vendor
tags
*.swp
.claude/
### OSX template
.DS_Store
.AppleDouble
@@ -142,6 +141,4 @@ test/s3/iam/.test_env
/test/erasure_coding/admin_dockertest/tmp
/test/erasure_coding/admin_dockertest/task_logs
weed_bin
telemetry/server/telemetry-server
.aider*
/seaweed-volume/docs
-5
View File
@@ -1,5 +0,0 @@
{
"setup": [],
"teardown": [],
"run": []
}
+3 -5
View File
@@ -1,4 +1,4 @@
.PHONY: test admin-generate admin-build admin-clean admin-dev admin-run admin-test admin-fmt admin-help weed-commands
.PHONY: test admin-generate admin-build admin-clean admin-dev admin-run admin-test admin-fmt admin-help
BINARY = weed
ADMIN_DIR = weed/admin
@@ -11,9 +11,6 @@ all: install
install: admin-generate
cd weed; go install
weed-commands:
cd weed && $(MAKE) weed-db weed-sql
warp_install:
go install github.com/minio/warp@v0.7.6
@@ -42,7 +39,8 @@ test: admin-generate
# Admin component targets
admin-generate:
@cd $(ADMIN_DIR) && $(MAKE) generate
@echo "Generating admin component templates..."
@templ generate
admin-build: admin-generate
@echo "Building admin component..."
+27 -52
View File
@@ -35,7 +35,6 @@ Your support will be really appreciated by me and other supporters!
[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
[![keepsec](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/keepsec.png)](https://www.keepsec.ca)
[![zyner](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_zyner.png)](https://zyner.org)
---
@@ -57,6 +56,7 @@ Table of Contents
* [Quick Start](#quick-start)
* [Quick Start with weed mini](#quick-start-with-weed-mini)
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
* [Quick Start with Single Binary](#quick-start-with-single-binary)
* [Introduction](#introduction)
* [Features](#features)
* [Additional Features](#additional-features)
@@ -80,41 +80,39 @@ Table of Contents
## Quick Start with weed mini ##
The easiest way to get started with SeaweedFS for development and testing:
Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file, or run `go install github.com/seaweedfs/seaweedfs/weed@latest`. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
Example:
```bash
AWS_ACCESS_KEY_ID=admin \
AWS_SECRET_ACCESS_KEY=secret \
S3_BUCKET=my-bucket \
# remove quarantine on macOS
# xattr -d com.apple.quarantine ./weed
./weed mini -dir=/data
```
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
The same command starts everything else too:
- **S3 Endpoint**: http://localhost:8333
This single command starts a complete SeaweedFS setup with:
- **Master UI**: http://localhost:9333
- **Volume Server**: http://localhost:9340
- **Filer UI**: http://localhost:8888
- **S3 Endpoint**: http://localhost:8333
- **WebDAV**: http://localhost:7333
- **Admin UI**: http://localhost:23646
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
Perfect for development, testing, learning SeaweedFS, and single-node deployments. To scale out, add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, on another machine, or on thousands of machines.
Perfect for development, testing, learning SeaweedFS, and single node deployments!
## Quick Start for S3 API on Docker ##
```bash
docker run -p 8333:8333 \
-e AWS_ACCESS_KEY_ID=admin \
-e AWS_SECRET_ACCESS_KEY=secret \
-e S3_BUCKET=my-bucket \
chrislusf/seaweedfs
```
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
Same behavior as the `weed mini` command above — the S3 endpoint is at http://localhost:8333 with `my-bucket` pre-created. Drop the env vars to run anonymously for development.
## Quick Start with Single Binary ##
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
# Introduction #
@@ -147,11 +145,6 @@ SeaweedFS can achieve both fast local access time and elastic cloud storage capa
What's more, the cloud storage access API cost is minimized.
Faster and cheaper than direct cloud storage!
SeaweedFS also ships a built-in **Iceberg REST Catalog**, turning the same cluster into a self-contained lakehouse.
Spark, Trino, Dremio, DuckDB, and RisingWave can query Iceberg tables directly — no Hive Metastore, Glue, or
external catalog service required. Storage and table metadata live in one system, simplifying on-prem and
small-team analytics stacks.
[Back to TOC](#table-of-contents)
# Features #
@@ -188,13 +181,6 @@ small-team analytics stacks.
* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
## Data Lakehouse Features ##
* [S3 Table Buckets][S3TableBucket] expose a dedicated namespace for Iceberg tables with strict layout validation.
* Built-in [Iceberg REST Catalog][IcebergCatalog] runs alongside the S3 endpoint — no external metastore needed.
* Native integrations with [Apache Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], and [RisingWave][RisingWaveIceberg].
* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan removal, manifest rewriting.
* Granular IAM at the bucket, namespace, and table level via standard S3 bucket policies.
## Kubernetes ##
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
@@ -218,14 +204,6 @@ small-team analytics stacks.
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
[S3TableBucket]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Table-Bucket
[IcebergCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Iceberg-Catalog
[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
[SparkIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Spark-Iceberg-Integration
[TrinoIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Trino-Iceberg-Integration
[DremioIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Dremio-Iceberg-Integration
[DuckDBIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/DuckDB-Iceberg-Integration
[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
[Back to TOC](#table-of-contents)
@@ -426,7 +404,7 @@ If the hot/warm data is split as 20/80, with 20 servers, you can achieve storage
## SeaweedFS Filer ##
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory structure is an interface that is implemented in many key-value stores or databases.
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory sturcture is an interface that is implemented in many key-value stores or databases.
The content of a file is mapped to one or many blobs, distributed to multiple volumes on multiple volume servers.
@@ -510,22 +488,20 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mong
### Compared to MinIO ###
Please note, as Apr 25, 2026 MinIO ceased development. It's strongly discouraged to use that unmaintained software with multiple security bugs.
MinIO follows AWS S3 closely and is ideal for testing for S3 API. It has good UI, policies, versionings, etc. SeaweedFS is trying to catch up here. It is also possible to put MinIO as a gateway in front of SeaweedFS later.
MinIO followed AWS S3 closely and was ideal for testing for S3 API. It had good UI, policies, versionings, etc. SeaweedFS is trying to catch up here.
MinIO metadata are in simple files. Each file write will incur extra writes to corresponding meta file.
MinIO metadata were in simple files. Each file write will incur extra writes to corresponding meta file.
MinIO did not have optimization for lots of small files. The files were simply stored as is to local disks.
MinIO does not have optimization for lots of small files. The files are simply stored as is to local disks.
Plus the extra meta file and shards for erasure coding, it only amplifies the LOSF problem.
MinIO had multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
MinIO has multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
MinIO had full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
MinIO has full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
MinIO did not have POSIX-like API support.
MinIO does not have POSIX-like API support.
MinIO had specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
MinIO has specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
## Dev Plan ##
@@ -655,8 +631,7 @@ Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
## Enterprise ##
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
which has advanced features, including data recovery, self-healing storage,
customizable erasure coding, EC vacuum and repair, etc.
which has a self-healing storage format with better data protection.
[Back to TOC](#table-of-contents)
File diff suppressed because it is too large Load Diff
-78
View File
@@ -1,78 +0,0 @@
# Security Policy
## Supported versions
Security fixes land in the latest release. Please reproduce against a recent
release or `master` before reporting; issues that only reproduce on old,
unsupported versions are not eligible for a fix or an advisory.
## Reporting a vulnerability
Report privately through GitHub private vulnerability reporting (the "Report a
vulnerability" button under the repository's Security tab). This keeps the
report, the fix, and any CVE in one place. Do not open a public issue.
### What a report must include
We can only act on reports that show real impact. Please include:
- Affected version (a release tag or `master` commit you reproduced on)
- The exact deployment and configuration: which components are running
(master, volume, filer, S3, admin), which ports are reachable by the
attacker, and what authentication is enabled
- The attacker's starting position: unauthenticated, a valid S3 user, an admin,
or someone with access to the internal cluster network
- The trust boundary that is crossed (e.g. an unauthenticated client reading
another tenant's data, an S3 user escalating to admin)
- A minimal, working reproduction or proof of concept
- Expected vs. actual behavior
A report without a working reproduction and a clear trust boundary is a
hardening suggestion, not a vulnerability. We are glad to receive those, but
they are handled on the normal issue tracker, not as security advisories.
### Automated and AI-assisted reports
Output from static analysis, dependency scanners, fuzzers, or LLMs is welcome
only when you have manually validated it and can supply a working reproduction
against a supported version, per the requirements above. Raw tool output,
speculative findings, or generated reports without a demonstrated exploit will
be closed as hardening suggestions.
## Trust model
SeaweedFS is built to run with its cluster components (master, volume servers,
and the raw filer API) on a trusted network. Those internal APIs are not an
authentication boundary unless you explicitly enable a control (for example
volume JWT or filer authentication) and that control is bypassed. Exposing an
internal port directly to untrusted clients is a deployment mistake, not a
vulnerability in SeaweedFS.
Reports are in scope when they cross a boundary SeaweedFS is meant to enforce,
for example:
- Unauthenticated access to data or operations that require authentication
- One S3 identity reading, writing, or deleting another identity's data
- Privilege escalation from a normal S3 user to administrative capability
- Bypass of Object Lock / retention where it is configured
- Remotely triggered data corruption or loss
Reports are generally out of scope when they require:
- Direct access to an internal cluster port that is meant to be private
- Full master, filer, or volume server access (already a full compromise)
- An insecure example configuration rather than a documented secure setup
- Local-only impact on a host the attacker already controls
## CVE assignment
When a report is confirmed, we publish an advisory and request the CVE through
GitHub. CVEs assigned by third parties without coordinating with us, or for
issues that do not cross a boundary described above, may be disputed.
## Response and disclosure
- We aim to acknowledge a valid report within a few business days.
- We will investigate, work on a fix, and coordinate a disclosure timeline
with you.
- Please allow time for a fix before any public disclosure.
-790
View File
@@ -1,790 +0,0 @@
# Execution Plan: SeaweedFS Volume Server — Go to Rust Port
## Scope Summary
| Component | Go Source | Lines (non-test) | Description |
|---|---|---|---|
| CLI & startup | `weed/command/volume.go` | 476 | ~40 CLI flags, server bootstrap |
| HTTP server + handlers | `weed/server/volume_server*.go` | 1,517 | Struct, routes, read/write/delete handlers |
| gRPC handlers | `weed/server/volume_grpc_*.go` | 3,073 | 40 RPC method implementations |
| Storage engine | `weed/storage/` | 15,271 | Volumes, needles, index, compaction, EC, backend |
| Protobuf definitions | `weed/pb/volume_server.proto` | 759 | Service + message definitions |
| Shared utilities | `weed/security/`, `weed/stats/`, `weed/util/` | ~2,000+ | JWT, TLS, metrics, helpers |
| **Total** | | **~23,000+** | |
## Rust Crate & Dependency Strategy
```
seaweed-volume/
├── Cargo.toml
├── build.rs # protobuf codegen
├── proto/
│ ├── volume_server.proto # copied from Go, adapted
│ └── remote.proto
├── src/
│ ├── main.rs # CLI entry point
│ ├── config.rs # CLI flags + config
│ ├── server/
│ │ ├── mod.rs
│ │ ├── volume_server.rs # VolumeServer struct + lifecycle
│ │ ├── http_handlers.rs # HTTP route dispatch
│ │ ├── http_read.rs # GET/HEAD handlers
│ │ ├── http_write.rs # POST/PUT handlers
│ │ ├── http_delete.rs # DELETE handler
│ │ ├── http_admin.rs # /status, /healthz, /ui
│ │ ├── grpc_service.rs # gRPC trait impl dispatch
│ │ ├── grpc_vacuum.rs
│ │ ├── grpc_copy.rs
│ │ ├── grpc_erasure_coding.rs
│ │ ├── grpc_tail.rs
│ │ ├── grpc_admin.rs
│ │ ├── grpc_read_write.rs
│ │ ├── grpc_batch_delete.rs
│ │ ├── grpc_scrub.rs
│ │ ├── grpc_tier.rs
│ │ ├── grpc_remote.rs
│ │ ├── grpc_query.rs
│ │ ├── grpc_state.rs
│ │ └── grpc_client_to_master.rs # heartbeat
│ ├── storage/
│ │ ├── mod.rs
│ │ ├── store.rs # Store (multi-disk manager)
│ │ ├── volume.rs # Volume struct + lifecycle
│ │ ├── volume_read.rs
│ │ ├── volume_write.rs
│ │ ├── volume_compact.rs
│ │ ├── volume_info.rs
│ │ ├── needle/
│ │ │ ├── mod.rs
│ │ │ ├── needle.rs # Needle struct + serialization
│ │ │ ├── needle_read.rs
│ │ │ ├── needle_write.rs
│ │ │ ├── needle_map.rs # in-memory NeedleMap
│ │ │ ├── needle_value.rs
│ │ │ └── crc.rs
│ │ ├── super_block.rs
│ │ ├── idx/
│ │ │ ├── mod.rs
│ │ │ └── idx.rs # .idx file format read/write
│ │ ├── needle_map_leveldb.rs
│ │ ├── types.rs # NeedleId, Offset, Size, DiskType
│ │ ├── disk_location.rs # DiskLocation per-directory
│ │ ├── erasure_coding/
│ │ │ ├── mod.rs
│ │ │ ├── ec_volume.rs
│ │ │ ├── ec_shard.rs
│ │ │ ├── ec_encoder.rs # Reed-Solomon encoding
│ │ │ └── ec_decoder.rs
│ │ └── backend/
│ │ ├── mod.rs
│ │ ├── disk.rs
│ │ └── s3_backend.rs # tiered storage to S3
│ ├── topology/
│ │ └── volume_layout.rs # replication placement
│ ├── security/
│ │ ├── mod.rs
│ │ ├── guard.rs # whitelist + JWT gate
│ │ ├── jwt.rs
│ │ └── tls.rs
│ ├── stats/
│ │ ├── mod.rs
│ │ └── metrics.rs # Prometheus counters/gauges
│ └── util/
│ ├── mod.rs
│ ├── grpc.rs
│ ├── http.rs
│ └── file.rs
└── tests/
├── integration/
│ ├── http_read_test.rs
│ ├── http_write_test.rs
│ ├── grpc_test.rs
│ └── storage_test.rs
└── unit/
├── needle_test.rs
├── idx_test.rs
├── super_block_test.rs
└── ec_test.rs
```
### Key Rust dependencies
| Purpose | Crate |
|---|---|
| Async runtime | `tokio` |
| gRPC | `tonic` + `prost` |
| HTTP server | `hyper` + `axum` |
| CLI parsing | `clap` (derive) |
| Prometheus metrics | `prometheus` |
| JWT | `jsonwebtoken` |
| TLS | `rustls` + `tokio-rustls` |
| LevelDB | `rusty-leveldb` or `rocksdb` |
| Reed-Solomon EC | `reed-solomon-erasure` |
| Logging | `tracing` + `tracing-subscriber` |
| Config (security.toml) | `toml` + `serde` |
| CRC32 | `crc32fast` |
| Memory-mapped files | `memmap2` |
---
## Phased Execution Plan
### Phase 1: Project Skeleton & Protobuf Codegen
**Goal:** Cargo project compiles, proto codegen works, CLI parses all flags.
**Steps:**
1.1. Create `seaweed-volume/Cargo.toml` with all dependencies listed above.
1.2. Copy `volume_server.proto` and `remote.proto` into `proto/`. Adjust package paths for Rust codegen.
1.3. Create `build.rs` using `tonic-build` to compile `.proto` files into Rust types.
1.4. Create `src/main.rs` with `clap` derive structs mirroring all 40 CLI flags from `weed/command/volume.go`:
- `--port` (default 8080)
- `--port.grpc` (default 0 → 10000+port)
- `--port.public` (default 0 → same as port)
- `--ip` (auto-detect)
- `--id` (default empty → ip:port)
- `--publicUrl`
- `--ip.bind`
- `--master` (default "localhost:9333")
- `--mserver` (deprecated compat)
- `--preStopSeconds` (default 10)
- `--idleTimeout` (default 30)
- `--dataCenter`
- `--rack`
- `--index` [memory|leveldb|leveldbMedium|leveldbLarge]
- `--disk` [hdd|ssd|<tag>]
- `--tags`
- `--dir` (default temp dir)
- `--dir.idx`
- `--max` (default "8")
- `--whiteList`
- `--minFreeSpacePercent` (default "1")
- `--minFreeSpace`
- `--images.fix.orientation` (default false)
- `--readMode` [local|proxy|redirect] (default "proxy")
- `--cpuprofile`
- `--memprofile`
- `--compactionMBps` (default 0)
- `--maintenanceMBps` (default 0)
- `--fileSizeLimitMB` (default 256)
- `--concurrentUploadLimitMB` (default 0)
- `--concurrentDownloadLimitMB` (default 0)
- `--pprof` (default false)
- `--metricsPort` (default 0)
- `--metricsIp`
- `--inflightUploadDataTimeout` (default 60s)
- `--inflightDownloadDataTimeout` (default 60s)
- `--hasSlowRead` (default true)
- `--readBufferSizeMB` (default 4)
- `--index.leveldbTimeout` (default 0)
- `--debug` (default false)
- `--debug.port` (default 6060)
1.5. Implement the same flag validation logic from `startVolumeServer()`:
- Parse comma-separated `--dir`, `--max`, `--minFreeSpace`, `--disk`, `--tags`
- Replicate single-value-to-all-dirs expansion
- Validate count matches between dirs and limits
- `--mserver` backward compat
1.6. **Test:** `cargo build` succeeds. `cargo run -- --help` shows all flags. Proto types generated.
**Verification:** Run with `--port 8080 --dir /tmp --master localhost:9333` — should parse without error and print config.
---
### Phase 2: Core Storage Types & On-Disk Format
**Goal:** Read and write the SeaweedFS needle/volume binary format bit-for-bit compatible with Go.
**Source files to port:**
- `weed/storage/types/needle_types.go` → `src/storage/types.rs`
- `weed/storage/needle/needle.go` → `src/storage/needle/needle.rs`
- `weed/storage/needle/needle_read.go` → `src/storage/needle/needle_read.rs`
- `weed/storage/needle/needle_write.go` (partial) → `src/storage/needle/needle_write.rs`
- `weed/storage/needle/crc.go` → `src/storage/needle/crc.rs`
- `weed/storage/needle/needle_value_map.go` → `src/storage/needle/needle_value.rs`
- `weed/storage/super_block/super_block.go` → `src/storage/super_block.rs`
- `weed/storage/idx/` → `src/storage/idx/`
**Steps:**
2.1. **Fundamental types** (`types.rs`):
- `NeedleId` (u64), `Offset` (u32 or u64 depending on version), `Size` (i32, negative = deleted)
- `Cookie` (u32)
- `DiskType` enum (HDD, SSD, Custom)
- Version constants (Version1=1, Version2=2, Version3=3, CurrentVersion=3)
- Byte serialization matching Go's `binary.BigEndian` encoding
2.2. **SuperBlock** (`super_block.rs`):
- 8-byte header: Version(1) + ReplicaPlacement(1) + TTL(2) + CompactRevision(2) + Reserved(2)
- `ReplicaPlacement` struct with same/diff rack/dc counts
- `TTL` struct with count + unit
- Read/write from first 8 bytes of `.dat` file
- Match exact byte layout from `super_block.go`
2.3. **Needle binary format** (`needle.rs`, `needle_read.rs`):
- Version 2/3 header: Cookie(4) + NeedleId(8) + Size(4)
- Body: Data, Flags, Name, Mime, PairsSize, Pairs, LastModified, TTL, Checksum, AppendAtNs, Padding
- CRC32 checksum (matching Go's `crc32.ChecksumIEEE`)
- Padding to 8-byte alignment
- Read path: read header → compute body length → read body → verify CRC
2.4. **Idx file format** (`idx/`):
- Fixed 16-byte records: NeedleId(8) + Offset(4) + Size(4)
- Sequential append-only file
- Walk/iterate all entries
- Binary search not used (loaded into memory map)
2.5. **NeedleMap (in-memory)** (`needle_map.rs`):
- HashMap<NeedleId, NeedleValue> where NeedleValue = {Offset, Size}
- Load from `.idx` file on volume mount
- Support Get, Set, Delete operations
- Track file count, deleted count, deleted byte count
2.6. **Tests:**
- Unit test: write a needle to bytes → read it back → verify fields match
- Unit test: write/read SuperBlock round-trip
- Unit test: write/read idx entries round-trip
- **Cross-compat test:** Use Go volume server to create a small volume with known data. Read it from Rust and verify all needles decoded correctly. (Keep test fixture `.dat`/`.idx` files in `tests/fixtures/`)
---
### Phase 3: Volume Struct & Lifecycle
**Goal:** Mount, read from, write to, and unmount a volume.
**Source files to port:**
- `weed/storage/volume.go` → `src/storage/volume.rs`
- `weed/storage/volume_read.go` → `src/storage/volume_read.rs`
- `weed/storage/volume_write.go` → `src/storage/volume_write.rs`
- `weed/storage/volume_loading.go`
- `weed/storage/volume_vacuum.go` → `src/storage/volume_compact.rs`
- `weed/storage/volume_info/volume_info.go` → `src/storage/volume_info.rs`
- `weed/storage/volume_super_block.go`
**Steps:**
3.1. **Volume struct** (`volume.rs`):
- Fields: Id, dir, dataFile, nm (NeedleMap), SuperBlock, readOnly, lastModifiedTs, lastCompactIndexOffset, lastCompactRevision
- `noWriteOrDelete` / `noWriteCanDelete` / `readOnly` state flags
- File handles for `.dat` file (read + append)
- Lock strategy: `RwLock` for concurrent reads, exclusive writes
3.2. **Volume loading** — exact logic from `volume_loading.go`:
- Open `.dat` file, read SuperBlock from first 8 bytes
- Load `.idx` file into NeedleMap
- Handle `.vif` (VolumeInfo) JSON sidecar file
- Set volume state based on SuperBlock + VolumeInfo
3.3. **Volume read** (`volume_read.rs`) — from `volume_read.go`:
- `ReadNeedle(needleId, cookie)`: lookup in NeedleMap → seek in .dat → read needle bytes → verify cookie + CRC → return data
- Handle deleted needles (Size < 0)
- `ReadNeedleBlob(offset, size)`: raw blob read
- `ReadNeedleMeta(needleId, offset, size)`: read metadata only
3.4. **Volume write** (`volume_write.rs`) — from `volume_write.go`:
- `WriteNeedle(needle)`: serialize needle → append to .dat → update .idx → update NeedleMap
- `DeleteNeedle(needleId)`: mark as deleted in NeedleMap + append tombstone to .idx
- File size limit check
- Concurrent write serialization (mutex on write path)
3.5. **Volume compaction** (`volume_compact.rs`) — from `volume_vacuum.go`:
- `CheckCompact()`: compute garbage ratio
- `Compact()`: create new .dat/.idx, copy only live needles, update compact revision
- `CommitCompact()`: rename compacted files over originals
- `CleanupCompact()`: remove temp files
- Throttle by `compactionBytePerSecond`
3.6. **Volume info** (`volume_info.rs`):
- Read/write `.vif` JSON sidecar
- VolumeInfo protobuf struct mapping
- Remote file references for tiered storage
3.7. **Tests:**
- Mount a volume, write 100 needles, read them all back, verify content
- Delete 50 needles, verify they return "deleted"
- Compact, verify only 50 remain, verify content
- Read Go-created volume fixtures
---
### Phase 4: Store (Multi-Volume, Multi-Disk Manager)
**Goal:** Manage multiple volumes across multiple disk directories.
**Source files to port:**
- `weed/storage/store.go` → `src/storage/store.rs`
- `weed/storage/disk_location.go` → `src/storage/disk_location.rs`
- `weed/storage/store_ec.go`
- `weed/storage/store_state.go`
**Steps:**
4.1. **DiskLocation** (`disk_location.rs`):
- Directory path, max volume count, min free space, disk type, tags
- Load all volumes from directory on startup
- Track free space, check writable
4.2. **Store** (`store.rs`):
- Vector of `DiskLocation`s
- `GetVolume(volumeId)` → lookup across all locations
- `HasVolume(volumeId)` check
- `AllocateVolume(...)` — create new volume in appropriate location
- `DeleteVolume(...)`, `MountVolume(...)`, `UnmountVolume(...)`
- `DeleteCollection(collection)` — delete all volumes of a collection
- Collect volume status for heartbeat
- `SetStopping()`, `Close()`
- Persistent state (maintenance mode) via `store_state.go`
4.3. **Store state** — `VolumeServerState` protobuf with maintenance flag, persisted to disk.
4.4. **Tests:**
- Create store with 2 dirs, allocate volumes in each, verify load balancing
- Mount/unmount/delete lifecycle
- State persistence across restart
---
### Phase 5: Erasure Coding
**Goal:** Full EC shard encode/decode/read/write/rebuild.
**Source files to port:**
- `weed/storage/erasure_coding/` (3,599 lines)
**Steps:**
5.1. **EC volume + shard structs** — `EcVolume`, `EcShard` with file handles for `.ec00`–`.ec13` shard files + `.ecx` index + `.ecj` journal.
5.2. **EC encoder** — Reed-Solomon 10+4 (configurable) encoding using `reed-solomon-erasure` crate:
- `VolumeEcShardsGenerate`: read .dat → split into data shards → compute parity → write .ec00-.ec13 + .ecx
5.3. **EC decoder/reader** — reconstruct data from any 10 of 14 shards:
- `EcShardRead`: read range from a specific shard
- Locate needle in EC volume via .ecx index
- Handle cross-shard needle reads
5.4. **EC shard operations:**
- Copy, delete, mount, unmount shards
- `VolumeEcShardsRebuild`: rebuild missing shards from remaining
- `VolumeEcShardsToVolume`: reconstruct .dat from EC shards
- `VolumeEcBlobDelete`: mark deleted in EC journal
- `VolumeEcShardsInfo`: report shard metadata
5.5. **Tests:**
- Encode a volume → verify 14 shards created
- Delete 4 shards → rebuild → verify data intact
- Read individual needles from EC volume
- Cross-compat with Go-generated EC shards
---
### Phase 6: Backend / Tiered Storage
**Goal:** Support tiered storage to remote backends (S3, etc).
**Source files to port:**
- `weed/storage/backend/` (1,850 lines)
**Steps:**
6.1. **Backend trait** — abstract `BackendStorage` trait with `ReadAt`, `WriteAt`, `Truncate`, `Close`, `Name`.
6.2. **Disk backend** — default local disk implementation.
6.3. **S3 backend** — upload .dat to S3, read ranges via S3 range requests.
6.4. **Tier move operations:**
- `VolumeTierMoveDatToRemote`: upload .dat to remote, optionally delete local
- `VolumeTierMoveDatFromRemote`: download .dat from remote
6.5. **Tests:**
- Disk backend read/write round-trip
- S3 backend with mock/localstack
---
### Phase 7: Security Layer
**Goal:** JWT authentication, whitelist guard, TLS configuration.
**Source files to port:**
- `weed/security/guard.go` → `src/security/guard.rs`
- `weed/security/jwt.go` → `src/security/jwt.rs`
- `weed/security/tls.go` → `src/security/tls.rs`
**Steps:**
7.1. **Guard** (`guard.rs`):
- Whitelist IP check (exact match on `r.RemoteAddr`)
- Wrap handlers with whitelist enforcement
- `UpdateWhiteList()` for live reload
7.2. **JWT** (`jwt.rs`):
- `SeaweedFileIdClaims` with `fid` field
- Sign with HMAC-SHA256
- Verify + decode with expiry check
- Separate signing keys for read vs write
- `GetJwt(request)` — extract from `Authorization: Bearer` header or `jwt` query param
7.3. **TLS** (`tls.rs`):
- Load server TLS cert/key for gRPC and HTTPS
- Load client TLS for mutual TLS
- Read from `security.toml` config (same format as Go's viper config)
7.4. **Tests:**
- JWT sign → verify round-trip
- JWT with wrong key → reject
- JWT with expired token → reject
- JWT fid mismatch → reject
- Whitelist allow/deny
---
### Phase 8: Prometheus Metrics
**Goal:** Export same metric names as Go for dashboard compatibility.
**Source files to port:**
- `weed/stats/metrics.go` (volume server counters/gauges/histograms)
**Steps:**
8.1. Define all Prometheus metrics matching Go names:
- `VolumeServerRequestCounter` (labels: method, status)
- `VolumeServerRequestHistogram` (labels: method)
- `VolumeServerInFlightRequestsGauge` (labels: method)
- `VolumeServerInFlightUploadSize`
- `VolumeServerInFlightDownloadSize`
- `VolumeServerConcurrentUploadLimit`
- `VolumeServerConcurrentDownloadLimit`
- `VolumeServerHandlerCounter` (labels: type — UploadLimitCond, DownloadLimitCond)
- Read/Write/Delete request counters
8.2. Metrics HTTP endpoint on `--metricsPort`.
8.3. Optional push-based metrics loop (`LoopPushingMetric`).
8.4. **Test:** Verify metric names and labels match Go output.
---
### Phase 9: HTTP Server & Handlers
**Goal:** All HTTP endpoints with exact same behavior as Go.
**Source files to port:**
- `weed/server/volume_server.go` → `src/server/volume_server.rs`
- `weed/server/volume_server_handlers.go` → `src/server/http_handlers.rs`
- `weed/server/volume_server_handlers_read.go` → `src/server/http_read.rs`
- `weed/server/volume_server_handlers_write.go` → `src/server/http_write.rs`
- `weed/server/volume_server_handlers_admin.go` → `src/server/http_admin.rs`
- `weed/server/volume_server_handlers_helper.go` (URL parsing, proxy, JSON responses)
- `weed/server/volume_server_handlers_ui.go` → `src/server/http_admin.rs`
**Steps:**
9.1. **URL path parsing** — from `handlers_helper.go`:
- Parse `/<vid>,<fid>` and `/<vid>/<fid>` patterns
- Extract volume ID, file ID, filename, ext
9.2. **Route dispatch** — from `privateStoreHandler` and `publicReadOnlyHandler`:
- `GET /` → `GetOrHeadHandler`
- `HEAD /` → `GetOrHeadHandler`
- `POST /` → `PostHandler` (whitelist gated)
- `PUT /` → `PostHandler` (whitelist gated)
- `DELETE /` → `DeleteHandler` (whitelist gated)
- `OPTIONS /` → CORS preflight
- `GET /status` → JSON status
- `GET /healthz` → health check
- `GET /ui/index.html` → HTML UI page
- Static resources (CSS/JS for UI)
9.3. **GET/HEAD handler** (`http_read.rs`) — from `handlers_read.go` (468 lines):
- JWT read authorization check
- Lookup needle by volume ID + needle ID + cookie
- ETag / If-None-Match / If-Modified-Since conditional responses
- Content-Type from stored MIME or filename extension
- Content-Disposition header
- Content-Encoding (gzip/zstd stored data)
- Range request support (HTTP 206 Partial Content)
- JPEG orientation fix (if configured)
- Proxy to replica on local miss (readMode=proxy)
- Redirect to replica (readMode=redirect)
- Download tracking (in-flight size accounting)
9.4. **POST/PUT handler** (`http_write.rs`) — from `handlers_write.go` (170 lines):
- JWT write authorization check
- Multipart form parsing
- Extract file data, filename, content type, TTL, last-modified
- Optional gzip/zstd compression
- Write needle to volume
- Replicate to peers (same logic as Go's `DistributedOperation`)
- Return JSON: {name, size, eTag, error}
9.5. **DELETE handler** — already in handlers.go:
- JWT authorization
- Delete from local volume
- Replicate delete to peers
- Return JSON result
9.6. **Admin handlers** (`http_admin.rs`):
- `/status` → JSON with volumes, version, disk status
- `/healthz` → 200 OK if serving
- `/ui/index.html` → HTML dashboard
9.7. **Concurrency limiting** — from `handlers.go`:
- Upload concurrency limit with `sync::Condvar` + timeout
- Download concurrency limit with proxy fallback to replicas
- HTTP 429 on timeout, 499 on client cancel
- Replication traffic bypasses upload limits
9.8. **Public port** — if configured, separate listener with read-only routes (GET/HEAD/OPTIONS only).
9.9. **Request ID middleware** — generate unique request ID per request.
9.10. **Tests:**
- Integration: start server → upload file via POST → GET it back → verify content
- Integration: upload → DELETE → GET returns 404
- Integration: conditional GET with ETag → 304
- Integration: range request → 206 with correct bytes
- Integration: exceed upload limit → 429
- Integration: whitelist enforcement
- Integration: JWT enforcement
---
### Phase 10: gRPC Service Implementation
**Goal:** All 40 gRPC methods with exact logic.
**Source files to port:**
- `weed/server/volume_grpc_admin.go` (380 lines)
- `weed/server/volume_grpc_vacuum.go` (124 lines)
- `weed/server/volume_grpc_copy.go` (636 lines)
- `weed/server/volume_grpc_copy_incremental.go` (66 lines)
- `weed/server/volume_grpc_read_write.go` (74 lines)
- `weed/server/volume_grpc_batch_delete.go` (124 lines)
- `weed/server/volume_grpc_tail.go` (140 lines)
- `weed/server/volume_grpc_erasure_coding.go` (619 lines)
- `weed/server/volume_grpc_scrub.go` (121 lines)
- `weed/server/volume_grpc_tier_upload.go` (98 lines)
- `weed/server/volume_grpc_tier_download.go` (85 lines)
- `weed/server/volume_grpc_remote.go` (95 lines)
- `weed/server/volume_grpc_query.go` (69 lines)
- `weed/server/volume_grpc_state.go` (26 lines)
- `weed/server/volume_grpc_read_all.go` (35 lines)
- `weed/server/volume_grpc_client_to_master.go` (325 lines)
**Steps (grouped by functional area):**
10.1. **Implement `tonic::Service` for `VolumeServer`** — the generated trait from proto.
10.2. **Admin RPCs** (`grpc_admin.rs`):
- `AllocateVolume` — create volume on appropriate disk location
- `VolumeMount` / `VolumeUnmount` / `VolumeDelete`
- `VolumeMarkReadonly` / `VolumeMarkWritable`
- `VolumeConfigure` — change replication
- `VolumeStatus` — return read-only, size, file counts
- `VolumeServerStatus` — disk statuses, memory, version, DC, rack
- `VolumeServerLeave` — deregister from master
- `DeleteCollection`
- `VolumeNeedleStatus` — get needle metadata by ID
- `Ping` — latency measurement
- `GetState` / `SetState` — maintenance mode
10.3. **Vacuum RPCs** (`grpc_vacuum.rs`):
- `VacuumVolumeCheck` — return garbage ratio
- `VacuumVolumeCompact` — stream progress (streaming response)
- `VacuumVolumeCommit` — finalize compaction
- `VacuumVolumeCleanup` — remove temp files
10.4. **Copy RPCs** (`grpc_copy.rs`):
- `VolumeCopy` — stream .dat/.idx from source to create local copy
- `VolumeSyncStatus` — return sync metadata
- `VolumeIncrementalCopy` — stream .dat delta since timestamp (streaming)
- `CopyFile` — generic file copy by extension (streaming)
- `ReceiveFile` — receive streamed file (client streaming)
- `ReadVolumeFileStatus` — return file timestamps and sizes
10.5. **Read/Write RPCs** (`grpc_read_write.rs`):
- `ReadNeedleBlob` — raw needle blob read
- `ReadNeedleMeta` — needle metadata
- `WriteNeedleBlob` — raw needle blob write
- `ReadAllNeedles` — stream all needles from volume(s) (streaming)
10.6. **Batch delete** (`grpc_batch_delete.rs`):
- `BatchDelete` — delete multiple file IDs, return per-ID results
10.7. **Tail RPCs** (`grpc_tail.rs`):
- `VolumeTailSender` — stream new needles since timestamp (streaming)
- `VolumeTailReceiver` — connect to another volume server and tail its changes
10.8. **Erasure coding RPCs** (`grpc_erasure_coding.rs`):
- `VolumeEcShardsGenerate` — generate EC shards from volume
- `VolumeEcShardsRebuild` — rebuild missing shards
- `VolumeEcShardsCopy` — copy shards from another server
- `VolumeEcShardsDelete` — delete EC shards
- `VolumeEcShardsMount` / `VolumeEcShardsUnmount`
- `VolumeEcShardRead` — read from EC shard (streaming)
- `VolumeEcBlobDelete` — mark blob deleted in EC volume
- `VolumeEcShardsToVolume` — reconstruct volume from EC shards
- `VolumeEcShardsInfo` — return shard metadata
10.9. **Scrub RPCs** (`grpc_scrub.rs`):
- `ScrubVolume` — integrity check volumes (INDEX / FULL / LOCAL modes)
- `ScrubEcVolume` — integrity check EC volumes
10.10. **Tier RPCs** (`grpc_tier.rs`):
- `VolumeTierMoveDatToRemote` — upload to remote backend (streaming progress)
- `VolumeTierMoveDatFromRemote` — download from remote (streaming progress)
10.11. **Remote storage** (`grpc_remote.rs`):
- `FetchAndWriteNeedle` — fetch from remote storage, write locally, replicate
10.12. **Query** (`grpc_query.rs`):
- `Query` — experimental CSV/JSON/Parquet select on stored data (streaming)
10.13. **Master heartbeat** (`grpc_client_to_master.rs`):
- `heartbeat()` background task — periodic gRPC stream to master
- Send: volume info, EC shard info, disk stats, has-no-space flags, deleted volumes
- Receive: volume size limit, leader address, metrics config
- Reconnect on failure with backoff
- `StopHeartbeat()` for graceful shutdown
10.14. **Tests:**
- Integration test per RPC: call via tonic client → verify response
- Streaming RPCs: verify all chunks received
- Error cases: invalid volume ID, non-existent volume, etc.
- Heartbeat: mock master gRPC server, verify registration
---
### Phase 11: Startup, Lifecycle & Graceful Shutdown
**Goal:** Full server startup matching Go's `runVolume()` and `startVolumeServer()`.
**Steps:**
11.1. **Startup sequence** (match `volume.go` exactly):
1. Load security configuration from `security.toml`
2. Start metrics server on metrics port
3. Parse folder/max/minFreeSpace/diskType/tags
4. Validate all directory writable
5. Resolve IP, bind IP, public URL, gRPC port
6. Create `VolumeServer` struct
7. Check with master (initial handshake)
8. Create `Store` (loads all existing volumes from disk)
9. Create security `Guard`
10. Register HTTP routes on admin mux
11. Optionally register public mux
12. Start gRPC server on gRPC port
13. Start public HTTP server (if separated)
14. Start cluster HTTP server (with optional TLS)
15. Start heartbeat background task
16. Start metrics push loop
17. Register SIGHUP handler for config reload + new volume loading
11.2. **Graceful shutdown** (match Go exactly):
1. On SIGINT/SIGTERM:
2. Stop heartbeat (notify master we're leaving)
3. Wait `preStopSeconds`
4. Stop public HTTP server
5. Stop cluster HTTP server
6. Graceful stop gRPC server
7. `volumeServer.Shutdown()` → `store.Close()` (flush all volumes)
11.3. **Reload** (SIGHUP):
- Reload security config
- Update whitelist
- Load newly appeared volumes from disk
11.4. **Tests:**
- Start server → send SIGTERM → verify clean shutdown
- Start server → SIGHUP → verify config reloaded
---
### Phase 12: Integration & Cross-Compatibility Testing
**Goal:** Rust volume server is a drop-in replacement for Go volume server.
**Steps:**
12.1. **Binary compatibility tests:**
- Create volumes with Go volume server
- Start Rust volume server on same data directory
- Read all data → verify identical
- Write new data with Rust → read with Go → verify
12.2. **API compatibility tests:**
- Run same HTTP requests against both Go and Rust servers
- Compare response bodies, headers, status codes
- Test all gRPC RPCs against both
12.3. **Master interop test:**
- Start Go master server
- Register Rust volume server
- Verify heartbeat works
- Verify volume assignment works
- Upload via filer → stored on Rust volume server → read back
12.4. **Performance benchmarks:**
- Throughput: sequential writes, sequential reads
- Latency: p50/p99 for read/write
- Concurrency: parallel reads/writes
- Compare Rust vs Go numbers
12.5. **Edge cases:**
- Volume at max size
- Disk full handling
- Corrupt .dat file recovery
- Network partition during replication
- EC shard loss + rebuild
---
## Execution Order & Dependencies
```
Phase 1 (Skeleton + CLI) ← no deps, start here
↓
Phase 2 (Storage types) ← needs Phase 1 (types used everywhere)
↓
Phase 3 (Volume struct) ← needs Phase 2
↓
Phase 4 (Store manager) ← needs Phase 3
↓
Phase 7 (Security) ← independent, can parallel with 3-4
Phase 8 (Metrics) ← independent, can parallel with 3-4
↓
Phase 9 (HTTP server) ← needs Phase 4 + 7 + 8
Phase 10 (gRPC server) ← needs Phase 4 + 7 + 8
↓
Phase 5 (Erasure coding) ← needs Phase 4, wire into Phase 10
Phase 6 (Tiered storage) ← needs Phase 4, wire into Phase 10
↓
Phase 11 (Startup + shutdown) ← needs Phase 9 + 10
↓
Phase 12 (Integration tests) ← needs all above
```
## Estimated Scope
| Phase | Estimated Rust Lines | Complexity |
|---|---|---|
| 1. Skeleton + CLI | ~400 | Low |
| 2. Storage types | ~2,000 | High (binary compat critical) |
| 3. Volume struct | ~2,500 | High |
| 4. Store manager | ~1,000 | Medium |
| 5. Erasure coding | ~3,000 | High |
| 6. Tiered storage | ~1,500 | Medium |
| 7. Security | ~500 | Medium |
| 8. Metrics | ~300 | Low |
| 9. HTTP server | ~2,000 | High |
| 10. gRPC server | ~3,500 | High |
| 11. Startup/shutdown | ~500 | Medium |
| 12. Integration tests | ~2,000 | Medium |
| **Total** | **~19,000** | |
## Critical Invariants to Preserve
1. **Binary format compatibility** — Rust must read/write `.dat`, `.idx`, `.vif`, `.ecX` files identically to Go. A single byte off = data loss.
2. **gRPC wire compatibility** — Same proto, same field semantics. Go master must talk to Rust volume server seamlessly.
3. **HTTP API compatibility** — Same URL patterns, same JSON response shapes, same headers, same status codes.
4. **Replication protocol** — Write replication between Go and Rust volume servers must work bidirectionally.
5. **Heartbeat protocol** — Rust volume server must register with Go master and maintain heartbeat.
6. **CRC32 algorithm** — Must use IEEE polynomial (same as Go's `crc32.ChecksumIEEE`).
7. **JWT compatibility** — Tokens signed by Go filer/master must be verifiable by Rust volume server and vice versa.
-7
View File
@@ -1,7 +0,0 @@
package main
import "os"
func main() {
os.Exit(Run(os.Args[1:]))
}
-7
View File
@@ -1,7 +0,0 @@
package main
import "os"
func main() {
os.Exit(Run(os.Args[1:]))
}
-167
View File
@@ -1,167 +0,0 @@
# Design: Serializing Bucket Configuration Mutations
Issue #9651 — concurrent `PutBucketVersioning` + `PutBucketEncryption` (as Terraform
issues them in parallel) intermittently lose the encryption write.
## Root cause
The bucket's entire config lives in one filer entry, `/buckets/<name>`. Every
config API does a read-modify-write of that single entry, and the writes are not
serialized:
- `updateBucketConfig(bucket, fn)` (`s3api_bucket_config.go:468`) — sources from a
possibly-stale cached `BucketConfig`, mutates `Entry.Extended`, writes the
**whole** entry. Used by: versioning, object-lock config, lifecycle, ACL/owner.
- `UpdateBucketMetadata` → `setBucketMetadata` (`:1042`) — reads a fresh entry,
mutates `Entry.Content`, writes the **whole** entry. Used by: encryption, CORS,
tagging, ownership, policy, notification.
Two ingredients produce the lost update:
1. **No serialization** of the read→modify→write (the cache mutexes only guard the
in-memory map, not the RMW).
2. **Whole-entry rewrite from an independent snapshot** — `updateBucketConfig`
rebuilds from a stale cached `BucketConfig` whose `Content` predates the
concurrent encryption write, so writing the whole entry reverts `Content`.
Sequential calls always pass (each sees the previous write), so it only surfaces
under concurrency — and CI's slower IO widens the window (the "2 of ~12 runs").
## Goals
- No lost updates across concurrent bucket-config changes — for **all** config
fields, not just versioning/encryption.
- Correct for a single S3 gateway (the reported case) and for multiple gateways.
- Reuse the filer primitives just merged (per-path lock, `WriteCondition`,
`ObjectTransaction`); do not reintroduce a distributed lock.
- Minimal blast radius: the fix lands at the two chokepoint helpers.
## Non-goals
- Changing the one-entry-per-bucket storage model.
- Multi-filer-concurrent bucket writes (addressed only as an optional phase 3).
## The two ingredients map to two complementary fixes
### Fix A — serialize + read fresh (closes the window for whole-entry writers)
Both `updateBucketConfig` and `UpdateBucketMetadata` must run their RMW under one
per-bucket critical section, and **re-read the entry fresh from the filer inside
it** — not rebuild from the cached `BucketConfig`. The lock alone is insufficient:
without the fresh read, two serialized writers still each apply a stale snapshot.
### Fix B — field-level updates (removes the collision entirely)
The two writers touch disjoint fields (`Extended[versioning]` vs `Content`). If
each path updated only its own field instead of rewriting the whole entry, neither
could clobber the other regardless of ordering. This is the structural fix and
makes serialization a defense-in-depth concern rather than a correctness
requirement for cross-field cases.
## Where to serialize (layering)
The bucket entry is a single filer entry, so unlike object writes there is no
sharding — the question is purely the scope of the lock:
| Layer | Serializes across | Cost | Notes |
|---|---|---|---|
| 1. Gateway-local per-bucket lock | one gateway process | tiny | fixes the reported (single-gateway/CI) case |
| 2. Filer per-path lock via conditional write | all gateways on one filer | small | reuses #9640 `CreateEntry`+`WriteCondition` |
| 3. Route-by-key to bucket-key owner filer | all gateways and filers | medium | same mechanism as the object DLM-removal |
## Recommended plan (phased)
### Phase 1 — minimal fix for #9651 (gateway-local lock + fresh read)
Add a bounded per-bucket lock table to `S3ApiServer`, reusing the same
`util.LockTable` the filer uses for its per-path lock:
```go
// in S3ApiServer
bucketConfigLocks *util.LockTable[string] // serialize bucket-entry RMW
func (s3a *S3ApiServer) withBucketConfigLock(bucket string, fn func() s3err.ErrorCode) s3err.ErrorCode {
lk := s3a.bucketConfigLocks.AcquireLock("bucketConfig", bucket, util.ExclusiveLock)
defer s3a.bucketConfigLocks.ReleaseLock(bucket, lk)
return fn()
}
```
Wrap the RMW in **both** chokepoints, and inside the lock read the entry fresh:
- `updateBucketConfig`: acquire the lock; re-read `/buckets/<name>` from the filer
(not the cache); rebuild `BucketConfig` from that fresh entry; apply `fn`; write;
invalidate cache; release.
- `UpdateBucketMetadata`/`setBucketMetadata`: same lock key; it already reads fresh,
so it just needs to share the critical section.
Both must use the **same** lock keyed on `bucket`, so versioning and encryption
contend on one mutex. This closes the reported window. Limitation: only one
gateway; two gateways behind a load balancer still race.
Test: parallel `PutBucketVersioning` + `PutBucketEncryption`, assert both persist
(the exact Terraform scenario), plus an N-way parallel variant over distinct
fields.
### Phase 2 — robust across gateways (field-level + CAS via merged primitives)
Move the writers off whole-entry rewrites:
- **Extended-based config** (versioning, object-lock, ownership, tagging-in-Extended)
→ `ObjectTransaction` `PATCH_EXTENDED` on `/buckets/<name>`. The owner filer reads
the entry fresh under its per-path lock and merges only the named keys, so the
gateway never sends a whole-entry snapshot — this dissolves *both* ingredients for
these fields.
- **`Content`-based config** (encryption, CORS, tags blob) — **chosen and
implemented (b3): extend `PATCH_EXTENDED` with `set_content`.** Under the same
per-path lock the filer reads the entry fresh, merges extended attributes, and
replaces `Content`, preserving the rest. So a content write becomes a field-level
patch too — `setBucketMetadata` patches `Content`, `updateBucketConfig` patches
extended keys, and the two serialize on the lock instead of racing whole-entry
rewrites. This is cleaner than the alternatives below: no client-side retry, no
storage migration, and it reuses `ObjectTransaction`'s existing atomic lock.
- (b1, rejected) Conditional `CreateEntry` overwrite with `IF_ETAG_MATCH` + retry
(#9640): correct but needs client-side retry, and the bucket directory entry has
no reliable ETag to compare on.
- (b2, future) Migrate each per-feature config out of the single `Content` blob
into its own `Extended` key. Then even *intra-blob* writes (tags vs encryption)
stop racing. Larger migration; tracked separately.
Once all paths are field-level patches, the phase-1 gateway lock is unnecessary —
the filer enforces atomicity. (This is the path taken: phase 1 was skipped.)
### Phase 3 — multi-filer (only if needed)
If multiple filers can write `/buckets/<name>` concurrently, a filer-local per-path
lock no longer suffices. Route bucket-config writes to
`PrimaryForKey("/buckets/<name>")` (the lock-ring view) and serialize on that one
owner filer — the same route-by-key design used to take object writes off the DLM.
Overkill for rare config writes; include only if multi-filer bucket writes are real.
## Correctness summary
- Phase 1: all RMW for a bucket serialize within a gateway; the fresh read means the
second writer observes the first's change. Closes #9651 for single-gateway.
- Phase 2: `PATCH_EXTENDED` is atomic field-level merge at the filer (no snapshot);
CAS turns a concurrent `Content` write into a retry, enforced under the filer's
per-path lock — correct for any number of gateways sharing a filer.
- Phase 3: one owner filer serializes all writers — correct across filers too.
## Scope checklist (every path that RMWs the bucket entry)
All of these funnel through the two chokepoints, so fixing the chokepoints covers
them — but the fix must not leave any of them on an unserialized path:
- via `updateBucketConfig`: versioning, object-lock config, lifecycle, ACL/owner.
- via `UpdateBucketMetadata`/`setBucketMetadata`: encryption, CORS, tagging,
ownership controls, bucket policy, notification.
- bucket create/delete (`CreateEntry`/`DeleteEntry` of `/buckets/<name>`) already
go through the filer's per-path lock on `CreateEntry`; ensure they take the same
bucket lock if they also patch config.
## Cache rule (must document in code)
Under the lock, **read the entry from the filer, never rebuild from the cached
`BucketConfig`**. The cache is for reads; it must be invalidated on every write and
never be the source for an RMW. This is the single most important detail — the lock
without the fresh read does not fix the bug.
+3 -5
View File
@@ -2,15 +2,13 @@ FROM ubuntu:22.04
LABEL author="Chris Lu"
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
# Use faster mirrors and optimize package installation
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
# For testing purposes, running as root avoids permission complexities and dependency
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
apt-get -o Acquire::http::Timeout=15 update && \
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
--no-install-recommends \
--no-install-suggests \
curl \
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25 AS builder
FROM golang:1.24 AS builder
RUN apt-get update && \
apt-get install -y build-essential wget ca-certificates && \
+3 -47
View File
@@ -1,6 +1,4 @@
# Pin the builder to the host arch and cross-compile the (CGO-free) Go binary,
# so arm64/arm/386 targets skip QEMU emulation of the whole compile.
FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder
FROM golang:1.24-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/seaweedfs/
ARG BRANCH=${BRANCH:-master}
@@ -14,49 +12,13 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
git checkout $BRANCH) || \
(echo "ERROR: Branch/commit $BRANCH not found in repository" && \
echo "Available branches:" && git branch -a && exit 1))
ARG TARGETOS TARGETARCH TARGETVARIANT
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
&& export GOOS=$TARGETOS GOARCH=$TARGETARCH \
&& case "$TARGETARCH" in arm) export GOARM="${TARGETVARIANT#v}";; esac \
&& CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -o /go/bin/weed .
# Rust volume server: use pre-built binary from CI when available (placed in
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build
# for non-native architectures.
FROM alpine:3.23 as rust_builder
ARG TARGETARCH
ARG TAGS
COPY weed-volume-prebuilt/ /prebuilt/
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
WORKDIR /build/seaweed-volume
RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
echo "Using pre-built Rust binary for ${TARGETARCH}" && \
cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
if [ "$TAGS" = "5BytesOffset" ]; then \
cargo build --release; \
else \
cargo build --release --no-default-features; \
fi && \
cp target/release/weed-volume /weed-volume; \
else \
echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
touch /weed-volume; \
fi
# Pre-built binaries arrive via GitHub Actions artifacts, which drop the
# executable bit, so the copied file is 0644 and exec fails with "Permission
# denied". Restore it (no-op for the empty placeholder, which stays size 0).
RUN chmod 0755 /weed-volume
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
FROM alpine AS final
LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/
# Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms)
COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
@@ -65,8 +27,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
# To disable: docker run -e GODEBUG=fips140=off ...
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
RUN apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -98,8 +59,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -7
View File
@@ -7,8 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
COPY ./entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse curl su-exec && \
RUN apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -40,8 +39,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25 AS builder
FROM golang:1.24 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+2 -8
View File
@@ -1,4 +1,4 @@
FROM golang:1.25 AS builder
FROM golang:1.24 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
@@ -34,8 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse snappy gflags curl su-exec && \
RUN apk add --no-cache fuse snappy gflags curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -68,8 +67,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -2
View File
@@ -17,8 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
-6
View File
@@ -15,9 +15,3 @@ COPY tarantool /opt/tarantool/app
# build app
RUN tt build app
# the base image's default healthcheck assumes a single-instance layout and
# checks a control socket path that doesn't exist for this multi-instance
# app, so it never reports healthy; check instance status instead.
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=6 \
CMD tt status app | awk 'NR>2{if ($2!="RUNNING") exit 1} END{if (NR<=2) exit 1}'
-3
View File
@@ -127,9 +127,6 @@ test_tarantool: tags = tarantool
test_tarantool: build_tarantool_dev_env build
docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
test_keycloak_s3: build
docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up
clean:
rm ./weed
-1
View File
@@ -7,7 +7,6 @@
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
# Scripts are skipped while an admin server is connected.
scripts = """
lock
ec.encode -fullPercent=95 -quietFor=1h
@@ -1,53 +0,0 @@
{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "AKIAIOSFODNN7EXAMPLE",
"secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
},
{
"name": "steward",
"credentials": [
{
"accessKey": "steward-key",
"secretKey": "steward-secret"
}
],
"actions": [
"Read",
"List",
"Write"
]
},
{
"name": "le001",
"credentials": [
{
"accessKey": "le001-key",
"secretKey": "le001-secret"
}
],
"actions": [
"Read",
"List"
]
},
{
"name": "anonymous",
"actions": [
"Read"
]
}
]
}
-260
View File
@@ -1,260 +0,0 @@
#!/usr/bin/env bash
#
# Integration test: git clone & pull on a SeaweedFS FUSE mount.
#
# Verifies that the mount correctly supports git's file operations by:
# 1. Creating a bare repo on the mount (acts as a remote)
# 2. Cloning it, making commits, and pushing back to the mount
# 3. Cloning from the mount into a working directory also on the mount
# 4. Pushing additional commits to the bare repo
# 5. Checking out an older revision in the on-mount clone
# 6. Running git pull to fast-forward with real changes
# 7. Verifying file content integrity at each step
#
# Usage:
# bash test-git-on-mount.sh /path/to/mount/point
#
# The mount must already be running. All test artifacts are created under
# <mount>/git-test-<pid> and cleaned up on exit (unless TEST_KEEP=1).
#
set -euo pipefail
MOUNT_DIR="${1:?Usage: $0 <mount-dir>}"
TEST_DIR="$MOUNT_DIR/git-test-$$"
LOCAL_DIR=$(mktemp -d)
PASS=0
FAIL=0
cleanup() {
if [[ "${TEST_KEEP:-}" == "1" ]]; then
echo "TEST_KEEP=1 — leaving artifacts:"
echo " mount: $TEST_DIR"
echo " local: $LOCAL_DIR"
else
rm -rf "$TEST_DIR" 2>/dev/null || true
rm -rf "$LOCAL_DIR" 2>/dev/null || true
fi
}
trap cleanup EXIT
# --- helpers ---------------------------------------------------------------
pass() { PASS=$((PASS + 1)); echo " PASS: $1"; }
fail() { FAIL=$((FAIL + 1)); echo " FAIL: $1"; }
assert_file_contains() {
local file=$1 expected=$2 label=$3
if [[ -f "$file" ]] && grep -qF "$expected" "$file" 2>/dev/null; then
pass "$label"
else
fail "$label (expected '$expected' in $file)"
fi
}
assert_file_exists() {
local file=$1 label=$2
if [[ -f "$file" ]]; then
pass "$label"
else
fail "$label ($file not found)"
fi
}
assert_file_not_exists() {
local file=$1 label=$2
if [[ ! -f "$file" ]]; then
pass "$label"
else
fail "$label ($file should not exist)"
fi
}
assert_eq() {
local actual=$1 expected=$2 label=$3
if [[ "$actual" == "$expected" ]]; then
pass "$label"
else
fail "$label (expected '$expected', got '$actual')"
fi
}
# --- setup -----------------------------------------------------------------
echo "========================================"
echo " Git-on-mount integration test"
echo "========================================"
echo "Mount: $MOUNT_DIR"
echo "Test: $TEST_DIR"
echo "Local: $LOCAL_DIR"
echo ""
if ! mountpoint -q "$MOUNT_DIR" 2>/dev/null && [[ ! -d "$MOUNT_DIR" ]]; then
echo "ERROR: $MOUNT_DIR is not a valid directory"
exit 1
fi
mkdir -p "$TEST_DIR"
# --- Phase 1: Create bare repo on mount -----------------------------------
echo "--- Phase 1: Create bare repo on mount ---"
BARE_REPO="$TEST_DIR/repo.git"
git init --bare "$BARE_REPO" >/dev/null 2>&1
pass "bare repo created on mount"
# --- Phase 2: Clone locally, make initial commits, push -------------------
echo "--- Phase 2: Clone locally, make initial commits, push ---"
LOCAL_CLONE="$LOCAL_DIR/clone1"
git clone "$BARE_REPO" "$LOCAL_CLONE" >/dev/null 2>&1
cd "$LOCAL_CLONE"
git config user.email "test@seaweedfs.test"
git config user.name "SeaweedFS Test"
# Commit 1: initial files
echo "hello world" > README.md
mkdir -p src
echo 'package main; import "fmt"; func main() { fmt.Println("v1") }' > src/main.go
git add -A && git commit -m "initial commit" >/dev/null 2>&1
COMMIT1=$(git rev-parse HEAD)
# Commit 2: add more files
mkdir -p data
for i in $(seq 1 20); do
printf "file-%03d: %s\n" "$i" "$(head -c 64 /dev/urandom | base64)" > "data/file-$(printf '%03d' $i).txt"
done
git add -A && git commit -m "add data files" >/dev/null 2>&1
COMMIT2=$(git rev-parse HEAD)
# Commit 3: modify and add
echo 'package main; import "fmt"; func main() { fmt.Println("v2") }' > src/main.go
echo "# Updated readme" >> README.md
mkdir -p docs
echo "documentation content" > docs/guide.md
git add -A && git commit -m "update src and add docs" >/dev/null 2>&1
COMMIT3=$(git rev-parse HEAD)
git push origin master >/dev/null 2>&1 || git push origin main >/dev/null 2>&1
BRANCH=$(git rev-parse --abbrev-ref HEAD)
pass "3 commits pushed to mount bare repo (branch=$BRANCH)"
# --- Phase 3: Clone from mount bare repo to mount working dir -------------
echo "--- Phase 3: Clone from mount bare repo to on-mount working dir ---"
MOUNT_CLONE="$TEST_DIR/working"
git clone "$BARE_REPO" "$MOUNT_CLONE" >/dev/null 2>&1
# Verify clone integrity
assert_file_exists "$MOUNT_CLONE/README.md" "README.md exists after clone"
assert_file_contains "$MOUNT_CLONE/README.md" "# Updated readme" "README.md has latest content"
assert_file_contains "$MOUNT_CLONE/src/main.go" 'v2' "src/main.go has v2"
assert_file_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md exists"
assert_file_exists "$MOUNT_CLONE/data/file-001.txt" "data files exist"
assert_file_exists "$MOUNT_CLONE/data/file-020.txt" "data/file-020.txt exists"
CLONE_HEAD=$(cd "$MOUNT_CLONE" && git rev-parse HEAD)
assert_eq "$CLONE_HEAD" "$COMMIT3" "on-mount clone HEAD matches commit 3"
# Count files
FILE_COUNT=$(find "$MOUNT_CLONE/data" -name '*.txt' | wc -l | tr -d ' ')
assert_eq "$FILE_COUNT" "20" "data/ has 20 files"
# --- Phase 4: Push more commits from local clone --------------------------
echo "--- Phase 4: Push more commits from local clone ---"
cd "$LOCAL_CLONE"
# Commit 4: larger changes
for i in $(seq 21 50); do
printf "file-%03d: %s\n" "$i" "$(head -c 128 /dev/urandom | base64)" > "data/file-$(printf '%03d' $i).txt"
done
echo 'package main; import "fmt"; func main() { fmt.Println("v3") }' > src/main.go
git add -A && git commit -m "expand data and update to v3" >/dev/null 2>&1
COMMIT4=$(git rev-parse HEAD)
# Commit 5: rename and delete
git mv docs/guide.md docs/manual.md
git rm data/file-001.txt >/dev/null 2>&1
git commit -m "rename guide, remove file-001" >/dev/null 2>&1
COMMIT5=$(git rev-parse HEAD)
git push origin "$BRANCH" >/dev/null 2>&1
pass "2 more commits pushed (5 total)"
# --- Phase 5: Checkout older revision in on-mount clone -------------------
echo "--- Phase 5: Checkout older revision in on-mount clone ---"
cd "$MOUNT_CLONE"
git checkout "$COMMIT2" >/dev/null 2>&1
# Verify we're at commit 2 state
DETACHED_HEAD=$(git rev-parse HEAD)
assert_eq "$DETACHED_HEAD" "$COMMIT2" "on-mount clone at commit 2 (detached)"
assert_file_not_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md not in commit 2"
assert_file_contains "$MOUNT_CLONE/src/main.go" 'v1' "src/main.go has v1 at commit 2"
# --- Phase 6: Return to branch and pull -----------------------------------
echo "--- Phase 6: Return to branch and pull with real changes ---"
cd "$MOUNT_CLONE"
git checkout "$BRANCH" >/dev/null 2>&1
# At this point the on-mount clone is at commit 3, remote is at commit 5
OLD_HEAD=$(git rev-parse HEAD)
assert_eq "$OLD_HEAD" "$COMMIT3" "on-mount clone at commit 3 before pull"
git pull >/dev/null 2>&1
NEW_HEAD=$(git rev-parse HEAD)
assert_eq "$NEW_HEAD" "$COMMIT5" "HEAD matches commit 5 after pull"
# Verify commit 5 state
assert_file_contains "$MOUNT_CLONE/src/main.go" 'v3' "src/main.go has v3 after pull"
assert_file_exists "$MOUNT_CLONE/docs/manual.md" "docs/manual.md exists (renamed)"
assert_file_not_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md gone (renamed)"
assert_file_not_exists "$MOUNT_CLONE/data/file-001.txt" "data/file-001.txt removed"
assert_file_exists "$MOUNT_CLONE/data/file-050.txt" "data/file-050.txt exists"
FINAL_COUNT=$(find "$MOUNT_CLONE/data" -name '*.txt' | wc -l | tr -d ' ')
assert_eq "$FINAL_COUNT" "49" "data/ has 49 files (50 added, 1 removed)"
# --- Phase 7: Verify git log integrity -----------------------------------
echo "--- Phase 7: Verify git log integrity ---"
cd "$MOUNT_CLONE"
GIT_LOG=$(git log --format=%s)
LOG_COUNT=$(echo "$GIT_LOG" | wc -l | tr -d ' ')
assert_eq "$LOG_COUNT" "5" "git log shows 5 commits"
# Verify commit messages
echo "$GIT_LOG" | grep -qF "initial commit" && pass "commit 1 message in log" || fail "commit 1 message missing"
echo "$GIT_LOG" | grep -qF "expand data" && pass "commit 4 message in log" || fail "commit 4 message missing"
echo "$GIT_LOG" | grep -qF "rename guide" && pass "commit 5 message in log" || fail "commit 5 message missing"
# --- Phase 8: Verify git status is clean ----------------------------------
echo "--- Phase 8: Verify git status is clean ---"
cd "$MOUNT_CLONE"
STATUS=$(git status --porcelain)
if [[ -z "$STATUS" ]]; then
pass "git status is clean"
else
fail "git status has changes: $STATUS"
fi
# --- Results ---------------------------------------------------------------
echo ""
echo "========================================"
echo " Results: $PASS passed, $FAIL failed"
echo "========================================"
if [[ "$FAIL" -gt 0 ]]; then
exit 1
fi
-46
View File
@@ -1,46 +0,0 @@
services:
keycloak:
image: quay.io/keycloak/keycloak:26.0.7
command: ["start-dev", "--import-realm"]
environment:
KC_BOOTSTRAP_ADMIN_USERNAME: admin
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
KC_HTTP_ENABLED: "true"
KC_HOSTNAME: "http://keycloak:8080"
KC_HOSTNAME_STRICT: "false"
ports:
- "8080:8080"
volumes:
- ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro
healthcheck:
test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"]
interval: 10s
timeout: 5s
retries: 5
start_period: 60s
s3:
image: chrislusf/seaweedfs:local
command:
- -v=9
- server
- -ip=s3
- -filer
- -master.volumeSizeLimitMB=16
- -volume
- -volume.max=0
- -volume.preStopSeconds=1
- -s3
- -s3.port=8333
- -s3.config=/etc/seaweedfs/s3_config.json
- -s3.iam.config=/etc/seaweedfs/iam_config.json
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
volumes:
- ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro
- ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro
ports:
- "8333:8333"
depends_on:
keycloak:
condition: service_healthy
+7 -38
View File
@@ -20,38 +20,30 @@ if [ "$(id -u)" = "0" ]; then
DATA_UID=$(stat -c '%u' /data 2>/dev/null)
DATA_GID=$(stat -c '%g' /data 2>/dev/null)
# Only run chown -R if ownership doesn't already match (avoids expensive
# recursive chown on subsequent starts, and is a no-op on OpenShift when
# fsGroup has already set correct ownership on the PVC).
if [ "$DATA_UID" != "$SEAWEED_UID" ] || [ "$DATA_GID" != "$SEAWEED_GID" ]; then
# Only run chown -R if ownership doesn't match (much faster for subsequent starts)
echo "Fixing /data ownership for seaweed user (uid=$SEAWEED_UID, gid=$SEAWEED_GID)"
if ! chown -R seaweed:seaweed /data; then
echo "Warning: Failed to change ownership of /data. This may cause permission errors." >&2
echo "If /data is read-only or has mount issues, the application may fail to start." >&2
fi
fi
# Use su-exec to drop privileges and run as seaweed user
exec su-exec seaweed "$0" "$@"
fi
isArgPassed() {
# Match both `-flag` and `--flag` (and their `=value` forms): the Go fla9
# library accepts both, and users may pick either form on the CLI.
arg="$1"
argWithEqualSign="$1="
argDouble="-$1"
argDoubleWithEqualSign="-$1="
shift
while [ $# -gt 0 ]; do
passedArg="$1"
shift
case $passedArg in
"$arg"|"$argDouble")
"$arg")
return 0
;;
"$argWithEqualSign"*|"$argDoubleWithEqualSign"*)
"$argWithEqualSign"*)
return 0
;;
esac
@@ -62,7 +54,7 @@ isArgPassed() {
case "$1" in
'master')
ARGS="-mdir=/data -volumeSizeLimitMB=1024"
ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
shift
exec /usr/bin/weed -logtostderr=true master $ARGS $@
;;
@@ -76,38 +68,15 @@ case "$1" in
exec /usr/bin/weed -logtostderr=true volume $ARGS $@
;;
'volume-rust')
ARGS="-dir /data -max 0"
if isArgPassed "-max" "$@"; then
ARGS="-dir /data"
fi
shift
if [ ! -s /usr/bin/weed-volume ]; then
echo "Error: Rust volume server is not available on this platform ($(uname -m))." >&2
echo "Use 'volume' for the Go volume server instead." >&2
exit 1
fi
exec /usr/bin/weed-volume $ARGS $@
;;
'server')
ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
ARGS="-dir=/data -volume.max=0 -master.volumePreallocate -master.volumeSizeLimitMB=1024"
if isArgPassed "-volume.max" "$@"; then
ARGS="-dir=/data -master.volumeSizeLimitMB=1024"
ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
fi
shift
exec /usr/bin/weed -logtostderr=true server $ARGS $@
;;
'mini')
ARGS="-dir=/data"
if isArgPassed "-dir" "$@"; then
ARGS=""
fi
shift
exec /usr/bin/weed -logtostderr=true mini $ARGS $@
;;
'filer')
ARGS=""
shift
+3 -3
View File
@@ -33,7 +33,7 @@ isArgPassed() {
case "$1" in
'master')
ARGS="-mdir=/data -volumeSizeLimitMB=1024"
ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
shift
exec /usr/bin/weed -logtostderr=true master $ARGS "$@"
;;
@@ -48,9 +48,9 @@ case "$1" in
;;
'server')
ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
ARGS="-dir=/data -volume.max=0 -master.volumePreallocate -master.volumeSizeLimitMB=1024"
if isArgPassed "-volume.max" "$@"; then
ARGS="-dir=/data -master.volumeSizeLimitMB=1024"
ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
fi
shift
exec /usr/bin/weed -logtostderr=true server $ARGS "$@"
-9
View File
@@ -42,12 +42,3 @@ services:
- master
- volume
- filer
admin:
image: chrislusf/seaweedfs:dev # use a remote dev image
ports:
- 23646:23646
command: 'admin -master=master:9333'
depends_on:
- master
- volume
- filer
+2 -2
View File
@@ -46,7 +46,7 @@ local filer_metadata = {
delete_by_directory_idx = function(directory)
local space = box.space.filer_metadata
local index = space.index.directory_idx
-- for each found directory
-- for each finded directories
for _, tuple in index:pairs({ directory }, { iterator = 'EQ' }) do
space:delete({ tuple[1], tuple[3] })
end
@@ -64,7 +64,7 @@ local filer_metadata = {
end
-- init results
local results = {}
-- for each found directory
-- for each finded directories
for _, tuple in directory_idx:pairs({ dirPath }, { iterator = 'EQ' }) do
-- filter by name
if filter_filename_func(tuple[3]) then

Some files were not shown because too many files have changed in this diff Show More