From 2c2df751f5d790df22f1cee21093468aab90d410 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 25 Jun 2026 10:52:23 -0700 Subject: [PATCH] Perf CI: benchmark the Rust volume server and report memory usage (#10111) * ci: add per-process memory sampler for perf jobs Samples VmRSS once a second into a CSV and records peak VmHWM per process on stop. Linux only; reads /proc//status. * ci: run perf benchmarks on the Rust volume server and report memory Matrix the throughput and S3 jobs over go/rust volume servers, using a standalone master (plus filer for S3) and swapping only the volume binary so the two are directly comparable. Sample peak RSS in every job and surface it per impl in the run summary. * ci: harden mem sampler arg handling and peak fallback Guard against missing args under set -u, and fall back to the max RSS sampled when a process exits before VmHWM can be read. --- .github/workflows/performance.yml | 251 ++++++++++++++++++++++++++---- test/perf/mem_sample.sh | 62 ++++++++ 2 files changed, 283 insertions(+), 30 deletions(-) create mode 100755 test/perf/mem_sample.sh diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 8d492d9ca..610ddb39e 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -7,6 +7,8 @@ on: - '**/*.go' - 'go.mod' - 'go.sum' + - 'seaweed-volume/**' + - 'test/perf/**' - '.github/workflows/performance.yml' workflow_dispatch: inputs: @@ -48,6 +50,9 @@ concurrency: permissions: contents: read +env: + VOL_SIZE_LIMIT: "1024" + jobs: performance-profile: name: CPU and Heap Profile @@ -83,6 +88,11 @@ jobs: # give the volume server a moment to register with the master sleep 3 + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + - name: Capture profiles under load run: | DURATION="${{ github.event.inputs.profile_duration || '30' }}" @@ -96,6 +106,22 @@ jobs: go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS)" + echo '```' + if [ -f mem-profile.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Profile summary if: always() run: | @@ -125,13 +151,21 @@ jobs: cpu-top.txt heap-top.txt goroutine.txt + mem-profile.csv + mem-profile.csv.peak weed.log retention-days: 30 benchmark: - name: Throughput Benchmark + name: Throughput Benchmark (${{ matrix.impl }} volume) runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + impl: [go, rust] + env: + IMPL: ${{ matrix.impl }} steps: - name: Check out code uses: actions/checkout@v7 @@ -141,15 +175,41 @@ jobs: with: go-version-file: 'go.mod' + - name: Install protobuf compiler + if: matrix.impl == 'rust' + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + if: matrix.impl == 'rust' + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + if: matrix.impl == 'rust' + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust- + - name: Build weed run: go build -o weed_bin ./weed - - name: Start server + - name: Build Rust volume server + if: matrix.impl == 'rust' + run: cd seaweed-volume && cargo build --release + + - name: Start master and volume server run: | - mkdir -p ./perfdata - ./weed_bin -v=1 server -dir=./perfdata -volume.max=0 \ - -master.volumeSizeLimitMB=1024 > weed.log 2>&1 & - echo "WEED_PID=$!" >> "$GITHUB_ENV" + mkdir -p ./perfdata/master ./perfdata/vol + ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \ + -mdir=./perfdata/master -peers=none \ + -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \ + > master.log 2>&1 & + echo "MASTER_PID=$!" >> "$GITHUB_ENV" for i in $(seq 1 60); do if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then echo "master is ready" @@ -157,8 +217,33 @@ jobs: fi sleep 1 done + if [ "${IMPL}" = "rust" ]; then + ./seaweed-volume/target/release/weed-volume \ + --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \ + --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \ + > volume.log 2>&1 & + else + ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \ + -port=8080 -dir=./perfdata/vol -max=100 \ + > volume.log 2>&1 & + fi + echo "VOLUME_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:8080/status >/dev/null 2>&1; then + echo "volume server is ready" + break + fi + sleep 1 + done + # let the volume server register with the master via heartbeat sleep 3 + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-benchmark.csv \ + "master=${MASTER_PID}" "volume=${VOLUME_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + - name: Run throughput benchmark run: | N="${{ github.event.inputs.benchmark_files || '100000' }}" @@ -168,46 +253,75 @@ jobs: -c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt - name: Run Go micro-benchmarks + if: matrix.impl == 'go' continue-on-error: true run: | go test -run='^$' -bench=. -benchmem -benchtime=10x \ ./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \ 2>&1 | tee go-benchmarks.txt + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS, ${IMPL} volume)" + echo '```' + if [ -f mem-benchmark.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Benchmark summary if: always() run: | { - echo "## Throughput benchmark" + echo "## Throughput benchmark (${IMPL} volume)" echo '```' grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \ benchmark-results.txt 2>/dev/null || echo "no benchmark results captured" echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Stop server + - name: Stop processes if: always() - run: kill "${WEED_PID}" 2>/dev/null || true + run: | + kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true - - name: Show server log on failure + - name: Show logs on failure if: failure() - run: tail -200 weed.log || true + run: | + echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true + echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true - name: Upload benchmark results if: always() uses: actions/upload-artifact@v7 with: - name: benchmark-results-${{ github.run_number }} + name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }} path: | benchmark-results.txt go-benchmarks.txt - weed.log + mem-benchmark.csv + mem-benchmark.csv.peak + master.log + volume.log retention-days: 7 s3-benchmark: - name: S3 Read/Write Benchmark + name: S3 Read/Write Benchmark (${{ matrix.impl }} volume) runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + impl: [go, rust] + env: + IMPL: ${{ matrix.impl }} steps: - name: Check out code uses: actions/checkout@v7 @@ -217,19 +331,43 @@ jobs: with: go-version-file: 'go.mod' + - name: Install protobuf compiler + if: matrix.impl == 'rust' + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + if: matrix.impl == 'rust' + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + if: matrix.impl == 'rust' + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust- + - name: Build weed and S3 load tool run: | go build -o weed_bin ./weed go build -o s3bench ./test/s3/benchmark - - name: Start server with S3 gateway + - name: Build Rust volume server + if: matrix.impl == 'rust' + run: cd seaweed-volume && cargo build --release + + - name: Start cluster with S3 gateway run: | - mkdir -p ./perfdata - ./weed_bin -v=1 server -dir=./perfdata -s3 -filer \ - -volume.max=0 -master.volumeSizeLimitMB=1024 \ - -s3.port=8000 -s3.config=./docker/compose/s3.json \ - > weed.log 2>&1 & - echo "WEED_PID=$!" >> "$GITHUB_ENV" + mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer + ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \ + -mdir=./perfdata/master -peers=none \ + -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \ + > master.log 2>&1 & + echo "MASTER_PID=$!" >> "$GITHUB_ENV" for i in $(seq 1 60); do if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then echo "master is ready" @@ -237,6 +375,29 @@ jobs: fi sleep 1 done + if [ "${IMPL}" = "rust" ]; then + ./seaweed-volume/target/release/weed-volume \ + --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \ + --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \ + > volume.log 2>&1 & + else + ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \ + -port=8080 -dir=./perfdata/vol -max=100 \ + > volume.log 2>&1 & + fi + echo "VOLUME_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:8080/status >/dev/null 2>&1; then + echo "volume server is ready" + break + fi + sleep 1 + done + sleep 3 + ./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \ + -s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \ + > filer.log 2>&1 & + echo "FILER_PID=$!" >> "$GITHUB_ENV" for i in $(seq 1 30); do if nc -z localhost 8000 2>/dev/null; then echo "s3 gateway is ready" @@ -246,6 +407,12 @@ jobs: done sleep 2 + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-s3.csv \ + "master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + - name: Run S3 read/write benchmark run: | OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}" @@ -256,31 +423,55 @@ jobs: -objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \ 2>&1 | tee s3-benchmark-results.txt + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS, ${IMPL} volume)" + echo '```' + if [ -f mem-s3.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: S3 benchmark summary if: always() run: | { - echo "## S3 read/write benchmark" + echo "## S3 read/write benchmark (${IMPL} volume)" echo '```' grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \ s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured" echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Stop server + - name: Stop processes if: always() - run: kill "${WEED_PID}" 2>/dev/null || true + run: | + kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true - - name: Show server log on failure + - name: Show logs on failure if: failure() - run: tail -200 weed.log || true + run: | + echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true + echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true + echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true - name: Upload S3 benchmark results if: always() uses: actions/upload-artifact@v7 with: - name: s3-benchmark-results-${{ github.run_number }} + name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }} path: | s3-benchmark-results.txt - weed.log + mem-s3.csv + mem-s3.csv.peak + master.log + volume.log + filer.log retention-days: 7 diff --git a/test/perf/mem_sample.sh b/test/perf/mem_sample.sh new file mode 100755 index 000000000..503f2adce --- /dev/null +++ b/test/perf/mem_sample.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Sample resident memory of one or more processes once per second. +# Usage: mem_sample.sh [label=pid ...] +# +# Writes a per-second time series to and, on stop, the peak +# resident set (VmHWM) per process to .peak. Stop with SIGTERM/SIGINT; +# if a process already exited, its peak falls back to the max RSS sampled. +# Linux only (reads /proc//status). +set -u + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 [label=pid ...]" >&2 + exit 1 +fi + +out="$1"; shift + +labels=(); pids=(); peaks=() +for arg in "$@"; do + labels+=("${arg%%=*}") + pids+=("${arg#*=}") + peaks+=(0) +done + +record_peaks() { + local total=0 i hwm + : > "${out}.peak" + for i in "${!pids[@]}"; do + hwm="$(awk '/^VmHWM:/{print $2}' "/proc/${pids[$i]}/status" 2>/dev/null)" + if [ -z "$hwm" ] || [ "$hwm" -eq 0 ]; then + hwm="${peaks[$i]}" + fi + printf '%s\t%s\n' "${labels[$i]}" "$hwm" >> "${out}.peak" + total=$((total + hwm)) + done + printf 'total\t%s\n' "$total" >> "${out}.peak" + exit 0 +} +trap record_peaks TERM INT + +header="epoch" +for l in "${labels[@]}"; do header+=",${l}_rss_kb"; done +header+=",total_rss_kb" +echo "$header" > "$out" + +while true; do + line="$(date +%s)"; total=0 + for i in "${!pids[@]}"; do + p="${pids[$i]}" + rss="$(awk '/^VmRSS:/{print $2}' "/proc/$p/status" 2>/dev/null)" + rss="${rss:-0}" + if [ "$rss" -gt "${peaks[$i]}" ]; then + peaks[$i]="$rss" + fi + line+=",${rss}" + total=$((total + rss)) + done + echo "${line},${total}" >> "$out" + # Sleep via a background child + wait so a caught signal interrupts promptly. + sleep 1 & + wait $! 2>/dev/null +done