diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 8d492d9ca..610ddb39e 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -7,6 +7,8 @@ on: - '**/*.go' - 'go.mod' - 'go.sum' + - 'seaweed-volume/**' + - 'test/perf/**' - '.github/workflows/performance.yml' workflow_dispatch: inputs: @@ -48,6 +50,9 @@ concurrency: permissions: contents: read +env: + VOL_SIZE_LIMIT: "1024" + jobs: performance-profile: name: CPU and Heap Profile @@ -83,6 +88,11 @@ jobs: # give the volume server a moment to register with the master sleep 3 + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + - name: Capture profiles under load run: | DURATION="${{ github.event.inputs.profile_duration || '30' }}" @@ -96,6 +106,22 @@ jobs: go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS)" + echo '```' + if [ -f mem-profile.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Profile summary if: always() run: | @@ -125,13 +151,21 @@ jobs: cpu-top.txt heap-top.txt goroutine.txt + mem-profile.csv + mem-profile.csv.peak weed.log retention-days: 30 benchmark: - name: Throughput Benchmark + name: Throughput Benchmark (${{ matrix.impl }} volume) runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + impl: [go, rust] + env: + IMPL: ${{ matrix.impl }} steps: - name: Check out code uses: actions/checkout@v7 @@ -141,15 +175,41 @@ jobs: with: go-version-file: 'go.mod' + - name: Install protobuf compiler + if: matrix.impl == 'rust' + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + if: matrix.impl == 'rust' + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + if: matrix.impl == 'rust' + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust- + - name: Build weed run: go build -o weed_bin ./weed - - name: Start server + - name: Build Rust volume server + if: matrix.impl == 'rust' + run: cd seaweed-volume && cargo build --release + + - name: Start master and volume server run: | - mkdir -p ./perfdata - ./weed_bin -v=1 server -dir=./perfdata -volume.max=0 \ - -master.volumeSizeLimitMB=1024 > weed.log 2>&1 & - echo "WEED_PID=$!" >> "$GITHUB_ENV" + mkdir -p ./perfdata/master ./perfdata/vol + ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \ + -mdir=./perfdata/master -peers=none \ + -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \ + > master.log 2>&1 & + echo "MASTER_PID=$!" >> "$GITHUB_ENV" for i in $(seq 1 60); do if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then echo "master is ready" @@ -157,8 +217,33 @@ jobs: fi sleep 1 done + if [ "${IMPL}" = "rust" ]; then + ./seaweed-volume/target/release/weed-volume \ + --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \ + --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \ + > volume.log 2>&1 & + else + ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \ + -port=8080 -dir=./perfdata/vol -max=100 \ + > volume.log 2>&1 & + fi + echo "VOLUME_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:8080/status >/dev/null 2>&1; then + echo "volume server is ready" + break + fi + sleep 1 + done + # let the volume server register with the master via heartbeat sleep 3 + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-benchmark.csv \ + "master=${MASTER_PID}" "volume=${VOLUME_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + - name: Run throughput benchmark run: | N="${{ github.event.inputs.benchmark_files || '100000' }}" @@ -168,46 +253,75 @@ jobs: -c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt - name: Run Go micro-benchmarks + if: matrix.impl == 'go' continue-on-error: true run: | go test -run='^$' -bench=. -benchmem -benchtime=10x \ ./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \ 2>&1 | tee go-benchmarks.txt + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS, ${IMPL} volume)" + echo '```' + if [ -f mem-benchmark.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Benchmark summary if: always() run: | { - echo "## Throughput benchmark" + echo "## Throughput benchmark (${IMPL} volume)" echo '```' grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \ benchmark-results.txt 2>/dev/null || echo "no benchmark results captured" echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Stop server + - name: Stop processes if: always() - run: kill "${WEED_PID}" 2>/dev/null || true + run: | + kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true - - name: Show server log on failure + - name: Show logs on failure if: failure() - run: tail -200 weed.log || true + run: | + echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true + echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true - name: Upload benchmark results if: always() uses: actions/upload-artifact@v7 with: - name: benchmark-results-${{ github.run_number }} + name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }} path: | benchmark-results.txt go-benchmarks.txt - weed.log + mem-benchmark.csv + mem-benchmark.csv.peak + master.log + volume.log retention-days: 7 s3-benchmark: - name: S3 Read/Write Benchmark + name: S3 Read/Write Benchmark (${{ matrix.impl }} volume) runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + impl: [go, rust] + env: + IMPL: ${{ matrix.impl }} steps: - name: Check out code uses: actions/checkout@v7 @@ -217,19 +331,43 @@ jobs: with: go-version-file: 'go.mod' + - name: Install protobuf compiler + if: matrix.impl == 'rust' + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + if: matrix.impl == 'rust' + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + if: matrix.impl == 'rust' + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust- + - name: Build weed and S3 load tool run: | go build -o weed_bin ./weed go build -o s3bench ./test/s3/benchmark - - name: Start server with S3 gateway + - name: Build Rust volume server + if: matrix.impl == 'rust' + run: cd seaweed-volume && cargo build --release + + - name: Start cluster with S3 gateway run: | - mkdir -p ./perfdata - ./weed_bin -v=1 server -dir=./perfdata -s3 -filer \ - -volume.max=0 -master.volumeSizeLimitMB=1024 \ - -s3.port=8000 -s3.config=./docker/compose/s3.json \ - > weed.log 2>&1 & - echo "WEED_PID=$!" >> "$GITHUB_ENV" + mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer + ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \ + -mdir=./perfdata/master -peers=none \ + -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \ + > master.log 2>&1 & + echo "MASTER_PID=$!" >> "$GITHUB_ENV" for i in $(seq 1 60); do if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then echo "master is ready" @@ -237,6 +375,29 @@ jobs: fi sleep 1 done + if [ "${IMPL}" = "rust" ]; then + ./seaweed-volume/target/release/weed-volume \ + --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \ + --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \ + > volume.log 2>&1 & + else + ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \ + -port=8080 -dir=./perfdata/vol -max=100 \ + > volume.log 2>&1 & + fi + echo "VOLUME_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:8080/status >/dev/null 2>&1; then + echo "volume server is ready" + break + fi + sleep 1 + done + sleep 3 + ./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \ + -s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \ + > filer.log 2>&1 & + echo "FILER_PID=$!" >> "$GITHUB_ENV" for i in $(seq 1 30); do if nc -z localhost 8000 2>/dev/null; then echo "s3 gateway is ready" @@ -246,6 +407,12 @@ jobs: done sleep 2 + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-s3.csv \ + "master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + - name: Run S3 read/write benchmark run: | OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}" @@ -256,31 +423,55 @@ jobs: -objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \ 2>&1 | tee s3-benchmark-results.txt + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS, ${IMPL} volume)" + echo '```' + if [ -f mem-s3.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: S3 benchmark summary if: always() run: | { - echo "## S3 read/write benchmark" + echo "## S3 read/write benchmark (${IMPL} volume)" echo '```' grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \ s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured" echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Stop server + - name: Stop processes if: always() - run: kill "${WEED_PID}" 2>/dev/null || true + run: | + kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true - - name: Show server log on failure + - name: Show logs on failure if: failure() - run: tail -200 weed.log || true + run: | + echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true + echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true + echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true - name: Upload S3 benchmark results if: always() uses: actions/upload-artifact@v7 with: - name: s3-benchmark-results-${{ github.run_number }} + name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }} path: | s3-benchmark-results.txt - weed.log + mem-s3.csv + mem-s3.csv.peak + master.log + volume.log + filer.log retention-days: 7 diff --git a/test/perf/mem_sample.sh b/test/perf/mem_sample.sh new file mode 100755 index 000000000..503f2adce --- /dev/null +++ b/test/perf/mem_sample.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Sample resident memory of one or more processes once per second. +# Usage: mem_sample.sh [label=pid ...] +# +# Writes a per-second time series to and, on stop, the peak +# resident set (VmHWM) per process to .peak. Stop with SIGTERM/SIGINT; +# if a process already exited, its peak falls back to the max RSS sampled. +# Linux only (reads /proc//status). +set -u + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 [label=pid ...]" >&2 + exit 1 +fi + +out="$1"; shift + +labels=(); pids=(); peaks=() +for arg in "$@"; do + labels+=("${arg%%=*}") + pids+=("${arg#*=}") + peaks+=(0) +done + +record_peaks() { + local total=0 i hwm + : > "${out}.peak" + for i in "${!pids[@]}"; do + hwm="$(awk '/^VmHWM:/{print $2}' "/proc/${pids[$i]}/status" 2>/dev/null)" + if [ -z "$hwm" ] || [ "$hwm" -eq 0 ]; then + hwm="${peaks[$i]}" + fi + printf '%s\t%s\n' "${labels[$i]}" "$hwm" >> "${out}.peak" + total=$((total + hwm)) + done + printf 'total\t%s\n' "$total" >> "${out}.peak" + exit 0 +} +trap record_peaks TERM INT + +header="epoch" +for l in "${labels[@]}"; do header+=",${l}_rss_kb"; done +header+=",total_rss_kb" +echo "$header" > "$out" + +while true; do + line="$(date +%s)"; total=0 + for i in "${!pids[@]}"; do + p="${pids[$i]}" + rss="$(awk '/^VmRSS:/{print $2}' "/proc/$p/status" 2>/dev/null)" + rss="${rss:-0}" + if [ "$rss" -gt "${peaks[$i]}" ]; then + peaks[$i]="$rss" + fi + line+=",${rss}" + total=$((total + rss)) + done + echo "${line},${total}" >> "$out" + # Sleep via a background child + wait so a caught signal interrupts promptly. + sleep 1 & + wait $! 2>/dev/null +done