mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-21 14:46:58 +00:00
Chris LuandGitHub
baead6901c
Every workflow that builds the Rust volume server first installed protoc from a package manager - twelve steps across apt, brew and choco. That is 37s per job on a good day, and this week archive.ubuntu.com stalled long enough for four jobs to burn their whole timeout without reaching a build. protoc-bin-vendored ships the compiler as a build-dependency, so it now arrives through the cargo registry the workflows already cache and there is nothing left to install. cargo build works on a machine with no protoc at all, which is worth as much locally as it is in CI. It also pins the version. The apt protoc on ubuntu-22.04 is 3.12, old enough to reject proto3 optional, which is why build.rs passes --experimental_allow_proto3_optional; the vendored one is 31.1. The flag stays, since it costs nothing and keeps a build against an older PROTOC working, and an explicit PROTOC still overrides the vendored binary for packagers who supply their own.
236 lines
8.6 KiB
YAML
236 lines
8.6 KiB
YAML
name: "Rust Volume Server Tests"
|
|
|
|
on:
|
|
pull_request:
|
|
branches: [ master ]
|
|
paths:
|
|
- 'seaweed-volume/**'
|
|
- 'test/volume_server/**'
|
|
- 'weed/pb/volume_server.proto'
|
|
- 'weed/pb/volume_server_pb/**'
|
|
- '.github/workflows/rust-volume-server-tests.yml'
|
|
push:
|
|
branches: [ master, main ]
|
|
paths:
|
|
- 'seaweed-volume/**'
|
|
- 'test/volume_server/**'
|
|
- 'weed/pb/volume_server.proto'
|
|
- 'weed/pb/volume_server_pb/**'
|
|
- '.github/workflows/rust-volume-server-tests.yml'
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
|
|
jobs:
|
|
rust-unit-tests:
|
|
name: Rust Unit Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
|
# target/ can carry C objects built against a different glibc than we link against.
|
|
- name: Fingerprint build toolchain
|
|
id: toolchain
|
|
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Cache cargo registry and target
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: |
|
|
~/.cargo/registry
|
|
~/.cargo/git
|
|
seaweed-volume/target
|
|
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
|
restore-keys: |
|
|
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
|
|
|
- name: Build Rust volume server
|
|
run: cd seaweed-volume && cargo build --release
|
|
|
|
- name: Run Rust unit tests
|
|
run: cd seaweed-volume && cargo test
|
|
|
|
rust-integration-tests:
|
|
name: Rust Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
|
# target/ can carry C objects built against a different glibc than we link against.
|
|
- name: Fingerprint build toolchain
|
|
id: toolchain
|
|
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Cache cargo registry and target
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: |
|
|
~/.cargo/registry
|
|
~/.cargo/git
|
|
seaweed-volume/target
|
|
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
|
restore-keys: |
|
|
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
|
|
|
- name: Build Go weed binary
|
|
run: |
|
|
cd weed
|
|
go build -tags 5BytesOffset -o weed .
|
|
chmod +x weed
|
|
./weed version
|
|
|
|
- name: Build Rust volume binary
|
|
run: cd seaweed-volume && cargo build --release
|
|
|
|
- name: Run integration tests
|
|
env:
|
|
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
|
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
|
run: |
|
|
echo "Running Rust volume server integration tests..."
|
|
go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
|
|
|
|
- name: Collect logs on failure
|
|
if: failure()
|
|
run: |
|
|
mkdir -p /tmp/rust-volume-server-it-logs
|
|
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
|
|
|
|
- name: Archive logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rust-volume-server-integration-test-logs
|
|
path: /tmp/rust-volume-server-it-logs/
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
- name: Test summary
|
|
if: always()
|
|
run: |
|
|
echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
rust-volume-go-tests:
|
|
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 45
|
|
env:
|
|
# Keep in step with the length of matrix.shard below.
|
|
SHARD_COUNT: 3
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
test-type: [grpc, http]
|
|
shard: [1, 2, 3]
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
|
# target/ can carry C objects built against a different glibc than we link against.
|
|
- name: Fingerprint build toolchain
|
|
id: toolchain
|
|
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Cache cargo registry and target
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: |
|
|
~/.cargo/registry
|
|
~/.cargo/git
|
|
seaweed-volume/target
|
|
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
|
restore-keys: |
|
|
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
|
|
|
- name: Build Go weed binary
|
|
run: |
|
|
cd weed
|
|
go build -tags 5BytesOffset -o weed .
|
|
chmod +x weed
|
|
./weed version
|
|
|
|
- name: Build Rust volume binary
|
|
run: cd seaweed-volume && cargo build --release
|
|
|
|
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
|
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
|
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
|
- name: Select this shard's tests
|
|
env:
|
|
TEST_TYPE: ${{ matrix.test-type }}
|
|
SHARD: ${{ matrix.shard }}
|
|
run: |
|
|
tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
|
# An empty list would make -run match nothing and the shard pass vacuously.
|
|
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
|
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
|
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
|
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
|
|
|
- name: Run volume server integration tests with Rust volume
|
|
env:
|
|
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
|
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
|
VOLUME_SERVER_IMPL: rust
|
|
run: |
|
|
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
|
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
|
|
|
- name: Collect logs on failure
|
|
if: failure()
|
|
run: |
|
|
mkdir -p /tmp/rust-volume-go-test-logs
|
|
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
|
|
|
|
- name: Archive logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
|
|
path: /tmp/rust-volume-go-test-logs/
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
- name: Test summary
|
|
if: always()
|
|
run: |
|
|
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
|