From 3c549b33ab8d45356cbd05e031b93e376fe66ccc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 4 Aug 2026 21:42:34 -0700 Subject: [PATCH] ci: deal volume server tests across shards instead of bucketing by letter (#10576) Both workflows split the suite with ^Test[A-H] / ^Test[I-S] / ^Test[T-Z]. Test names cluster, so shard 2 drew 50 of the 114 grpc tests and 30 of the 64 http ones, and spent 13m51s against 7m48s and 9m09s for its peers. Listing the tests and dealing them out one at a time splits them 38/38/38 and 21/22/21, and keeps splitting evenly as tests are added. The pattern is computed once into the environment rather than repeated in the summary step, where the two copies had to be kept in agreement by hand. --- .../workflows/rust-volume-server-tests.yml | 56 +++++++----------- .../volume-server-integration-tests.yml | 57 +++++++------------ 2 files changed, 40 insertions(+), 73 deletions(-) diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml index 7626fd93a..6f2417296 100644 --- a/.github/workflows/rust-volume-server-tests.yml +++ b/.github/workflows/rust-volume-server-tests.yml @@ -146,6 +146,9 @@ jobs: name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) runs-on: ubuntu-22.04 timeout-minutes: 45 + env: + # Keep in step with the length of matrix.shard below. + SHARD_COUNT: 3 strategy: fail-fast: false matrix: @@ -194,30 +197,28 @@ jobs: - name: Build Rust volume binary run: cd seaweed-volume && cargo build --release + # Dealing the listed tests out one by one keeps the shards even. Bucketing + # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of + # the 114 grpc tests and ran nearly twice as long as the other two shards. + - name: Select this shard's tests + env: + TEST_TYPE: ${{ matrix.test-type }} + SHARD: ${{ matrix.shard }} + run: | + tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u) + # An empty list would make -run match nothing and the shard pass vacuously. + [ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; } + selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1') + echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests" + echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV" + - name: Run volume server integration tests with Rust volume env: WEED_BINARY: ${{ github.workspace }}/weed/weed RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume VOLUME_SERVER_IMPL: rust run: | - if [ "${{ matrix.test-type }}" == "grpc" ]; then - if [ "${{ matrix.shard }}" == "1" ]; then - TEST_PATTERN="^Test[A-H]" - elif [ "${{ matrix.shard }}" == "2" ]; then - TEST_PATTERN="^Test[I-S]" - else - TEST_PATTERN="^Test[T-Z]" - fi - else - if [ "${{ matrix.shard }}" == "1" ]; then - TEST_PATTERN="^Test[A-G]" - elif [ "${{ matrix.shard }}" == "2" ]; then - TEST_PATTERN="^Test[H-R]" - else - TEST_PATTERN="^Test[S-Z]" - fi - fi - echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..." + echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..." go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}" - name: Collect logs on failure @@ -238,23 +239,6 @@ jobs: - name: Test summary if: always() run: | - if [ "${{ matrix.test-type }}" == "grpc" ]; then - if [ "${{ matrix.shard }}" == "1" ]; then - TEST_PATTERN="^Test[A-H]" - elif [ "${{ matrix.shard }}" == "2" ]; then - TEST_PATTERN="^Test[I-S]" - else - TEST_PATTERN="^Test[T-Z]" - fi - else - if [ "${{ matrix.shard }}" == "1" ]; then - TEST_PATTERN="^Test[A-G]" - elif [ "${{ matrix.shard }}" == "2" ]; then - TEST_PATTERN="^Test[H-R]" - else - TEST_PATTERN="^Test[S-Z]" - fi - fi echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY" - echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY" echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml index 1ce31511a..c1874fa19 100644 --- a/.github/workflows/volume-server-integration-tests.yml +++ b/.github/workflows/volume-server-integration-tests.yml @@ -29,6 +29,8 @@ permissions: env: TEST_TIMEOUT: '30m' + # Keep in step with the length of matrix.shard below. + SHARD_COUNT: 3 jobs: volume-server-integration-tests: @@ -57,28 +59,26 @@ jobs: chmod +x weed ./weed version + # Dealing the listed tests out one by one keeps the shards even. Bucketing + # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of + # the 114 grpc tests and ran nearly twice as long as the other two shards. + - name: Select this shard's tests + env: + TEST_TYPE: ${{ matrix.test-type }} + SHARD: ${{ matrix.shard }} + run: | + tests=$(go test ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u) + # An empty list would make -run match nothing and the shard pass vacuously. + [ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; } + selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1') + echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests" + echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV" + - name: Run volume server integration tests env: WEED_BINARY: ${{ github.workspace }}/weed/weed run: | - if [ "${{ matrix.test-type }}" == "grpc" ]; then - if [ "${{ matrix.shard }}" == "1" ]; then - TEST_PATTERN="^Test[A-H]" - elif [ "${{ matrix.shard }}" == "2" ]; then - TEST_PATTERN="^Test[I-S]" - else - TEST_PATTERN="^Test[T-Z]" - fi - else - if [ "${{ matrix.shard }}" == "1" ]; then - TEST_PATTERN="^Test[A-G]" - elif [ "${{ matrix.shard }}" == "2" ]; then - TEST_PATTERN="^Test[H-R]" - else - TEST_PATTERN="^Test[S-Z]" - fi - fi - echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..." + echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..." go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}" - name: Collect logs on failure @@ -99,23 +99,6 @@ jobs: - name: Test summary if: always() run: | - if [ "${{ matrix.test-type }}" == "grpc" ]; then - if [ "${{ matrix.shard }}" == "1" ]; then - TEST_PATTERN="^Test[A-H]" - elif [ "${{ matrix.shard }}" == "2" ]; then - TEST_PATTERN="^Test[I-S]" - else - TEST_PATTERN="^Test[T-Z]" - fi - else - if [ "${{ matrix.shard }}" == "1" ]; then - TEST_PATTERN="^Test[A-G]" - elif [ "${{ matrix.shard }}" == "2" ]; then - TEST_PATTERN="^Test[H-R]" - else - TEST_PATTERN="^Test[S-Z]" - fi - fi echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY" - echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY" - echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY" + echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"