mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-21 22:56:55 +00:00
Add performance CI (profiling, throughput, S3 read/write) (#10105)
* test: add self-contained S3 read/write load tool Concurrent PUT/GET against the S3 gateway, reporting requests/sec, transfer rate, and latency percentiles. Built on the aws-sdk-go-v2 client the S3 tests already use, so no extra benchmark binary is needed. * ci: add performance workflow Three parallel jobs: cpu/heap pprof of the server under write load, native throughput via weed benchmark plus the Go micro-benchmarks, and an S3 read/write benchmark against the gateway. Runs on push to master and manual dispatch with tunable duration, object count, size, and concurrency.
This commit is contained in:
Chris Lu
@@ -0,0 +1,286 @@
|
||||
name: "Performance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/performance.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
profile_duration:
|
||||
description: "CPU profiling duration in seconds"
|
||||
required: false
|
||||
default: "30"
|
||||
type: string
|
||||
benchmark_files:
|
||||
description: "Number of files for the throughput benchmark"
|
||||
required: false
|
||||
default: "100000"
|
||||
type: string
|
||||
benchmark_concurrency:
|
||||
description: "Concurrent read/write workers"
|
||||
required: false
|
||||
default: "16"
|
||||
type: string
|
||||
benchmark_size:
|
||||
description: "Simulated file size in bytes"
|
||||
required: false
|
||||
default: "1024"
|
||||
type: string
|
||||
s3_objects:
|
||||
description: "Number of objects for the S3 benchmark"
|
||||
required: false
|
||||
default: "20000"
|
||||
type: string
|
||||
s3_size:
|
||||
description: "S3 object size in bytes"
|
||||
required: false
|
||||
default: "4096"
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/performance
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
performance-profile:
|
||||
name: CPU and Heap Profile
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Start server with profiling enabled
|
||||
run: |
|
||||
mkdir -p ./perfdata
|
||||
./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \
|
||||
-s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \
|
||||
-s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> weed.log 2>&1 &
|
||||
echo "WEED_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# give the volume server a moment to register with the master
|
||||
sleep 3
|
||||
|
||||
- name: Capture profiles under load
|
||||
run: |
|
||||
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
|
||||
# drive write load so the sampled profile reflects real work
|
||||
./weed_bin benchmark -master=localhost:9333 -writeOnly \
|
||||
-c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 &
|
||||
echo "Sampling CPU profile for ${DURATION}s..."
|
||||
curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt
|
||||
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
|
||||
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
|
||||
|
||||
- name: Profile summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## CPU profile (top functions)"
|
||||
echo '```'
|
||||
head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop server
|
||||
if: always()
|
||||
run: kill "${WEED_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: tail -200 weed.log || true
|
||||
|
||||
- name: Upload profiles
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-profile-${{ github.run_number }}
|
||||
path: |
|
||||
cpu.pprof
|
||||
heap.pprof
|
||||
cpu-top.txt
|
||||
heap-top.txt
|
||||
goroutine.txt
|
||||
weed.log
|
||||
retention-days: 30
|
||||
|
||||
benchmark:
|
||||
name: Throughput Benchmark
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Start server
|
||||
run: |
|
||||
mkdir -p ./perfdata
|
||||
./weed_bin -v=1 server -dir=./perfdata -volume.max=0 \
|
||||
-master.volumeSizeLimitMB=1024 > weed.log 2>&1 &
|
||||
echo "WEED_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 3
|
||||
|
||||
- name: Run throughput benchmark
|
||||
run: |
|
||||
N="${{ github.event.inputs.benchmark_files || '100000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.benchmark_size || '1024' }}"
|
||||
./weed_bin benchmark -master=localhost:9333 \
|
||||
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
|
||||
|
||||
- name: Run Go micro-benchmarks
|
||||
continue-on-error: true
|
||||
run: |
|
||||
go test -run='^$' -bench=. -benchmem -benchtime=10x \
|
||||
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
|
||||
2>&1 | tee go-benchmarks.txt
|
||||
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Throughput benchmark"
|
||||
echo '```'
|
||||
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
|
||||
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop server
|
||||
if: always()
|
||||
run: kill "${WEED_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: tail -200 weed.log || true
|
||||
|
||||
- name: Upload benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: benchmark-results-${{ github.run_number }}
|
||||
path: |
|
||||
benchmark-results.txt
|
||||
go-benchmarks.txt
|
||||
weed.log
|
||||
retention-days: 7
|
||||
|
||||
s3-benchmark:
|
||||
name: S3 Read/Write Benchmark
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build weed and S3 load tool
|
||||
run: |
|
||||
go build -o weed_bin ./weed
|
||||
go build -o s3bench ./test/s3/benchmark
|
||||
|
||||
- name: Start server with S3 gateway
|
||||
run: |
|
||||
mkdir -p ./perfdata
|
||||
./weed_bin -v=1 server -dir=./perfdata -s3 -filer \
|
||||
-volume.max=0 -master.volumeSizeLimitMB=1024 \
|
||||
-s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> weed.log 2>&1 &
|
||||
echo "WEED_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z localhost 8000 2>/dev/null; then
|
||||
echo "s3 gateway is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 2
|
||||
|
||||
- name: Run S3 read/write benchmark
|
||||
run: |
|
||||
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.s3_size || '4096' }}"
|
||||
./s3bench -endpoint=http://localhost:8000 \
|
||||
-access-key=some_access_key1 -secret-key=some_secret_key1 \
|
||||
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
|
||||
2>&1 | tee s3-benchmark-results.txt
|
||||
|
||||
- name: S3 benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## S3 read/write benchmark"
|
||||
echo '```'
|
||||
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
|
||||
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop server
|
||||
if: always()
|
||||
run: kill "${WEED_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: tail -200 weed.log || true
|
||||
|
||||
- name: Upload S3 benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-benchmark-results-${{ github.run_number }}
|
||||
path: |
|
||||
s3-benchmark-results.txt
|
||||
weed.log
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,223 @@
|
||||
// Command s3_benchmark drives concurrent PUT and GET load against an S3
|
||||
// gateway and reports throughput and latency. It is a self-contained load
|
||||
// generator for the performance CI, using the aws-sdk-go-v2 client the rest
|
||||
// of the S3 tests already depend on.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
func main() {
|
||||
endpoint := flag.String("endpoint", "http://localhost:8000", "S3 gateway endpoint")
|
||||
accessKey := flag.String("access-key", "some_access_key1", "S3 access key")
|
||||
secretKey := flag.String("secret-key", "some_secret_key1", "S3 secret key")
|
||||
region := flag.String("region", "us-east-1", "S3 region")
|
||||
bucket := flag.String("bucket", "perf-benchmark", "bucket to write into")
|
||||
objects := flag.Int("objects", 10000, "number of objects")
|
||||
size := flag.Int("size", 1024, "object size in bytes")
|
||||
concurrency := flag.Int("concurrency", 16, "concurrent workers")
|
||||
mode := flag.String("mode", "both", "write, read, or both")
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(context.Background(),
|
||||
config.WithRegion(*region),
|
||||
config.WithRetryMaxAttempts(1),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(*accessKey, *secretKey, "")),
|
||||
)
|
||||
if err != nil {
|
||||
fatalf("load aws config: %v", err)
|
||||
}
|
||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(*endpoint)
|
||||
o.UsePathStyle = true
|
||||
})
|
||||
|
||||
if err := ensureBucket(client, *bucket); err != nil {
|
||||
fatalf("ensure bucket: %v", err)
|
||||
}
|
||||
|
||||
payload := make([]byte, *size)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i)
|
||||
}
|
||||
|
||||
doWrite := *mode == "both" || *mode == "write"
|
||||
doRead := *mode == "both" || *mode == "read"
|
||||
|
||||
if doWrite {
|
||||
report("S3 WRITE", *concurrency, *size, run(*concurrency, *objects, func(key string) error {
|
||||
_, err := client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: bucket,
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader(payload),
|
||||
ContentLength: aws.Int64(int64(*size)),
|
||||
})
|
||||
return err
|
||||
}))
|
||||
}
|
||||
|
||||
if doRead {
|
||||
report("S3 READ", *concurrency, *size, run(*concurrency, *objects, func(key string) error {
|
||||
out, err := client.GetObject(context.Background(), &s3.GetObjectInput{
|
||||
Bucket: bucket,
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Body.Close()
|
||||
_, err = io.Copy(io.Discard, out.Body)
|
||||
return err
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
type result struct {
|
||||
completed int64
|
||||
failed int64
|
||||
elapsed time.Duration
|
||||
latencies []time.Duration
|
||||
firstErr error
|
||||
}
|
||||
|
||||
// run fans `total` keyed operations across `workers` goroutines, timing each.
|
||||
func run(workers, total int, op func(key string) error) result {
|
||||
var (
|
||||
completed, failed, nextKey atomic.Int64
|
||||
mu sync.Mutex
|
||||
firstErr error
|
||||
latencies = make([]time.Duration, 0, total)
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
start := time.Now()
|
||||
for w := 0; w < workers; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
local := make([]time.Duration, 0, total/workers+1)
|
||||
for {
|
||||
i := nextKey.Add(1) - 1
|
||||
if i >= int64(total) {
|
||||
break
|
||||
}
|
||||
key := fmt.Sprintf("obj-%d", i)
|
||||
opStart := time.Now()
|
||||
err := op(key)
|
||||
local = append(local, time.Since(opStart))
|
||||
if err != nil {
|
||||
failed.Add(1)
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
continue
|
||||
}
|
||||
completed.Add(1)
|
||||
}
|
||||
mu.Lock()
|
||||
latencies = append(latencies, local...)
|
||||
mu.Unlock()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return result{
|
||||
completed: completed.Load(),
|
||||
failed: failed.Load(),
|
||||
elapsed: time.Since(start),
|
||||
latencies: latencies,
|
||||
firstErr: firstErr,
|
||||
}
|
||||
}
|
||||
|
||||
func report(label string, concurrency, size int, r result) {
|
||||
secs := r.elapsed.Seconds()
|
||||
transferred := r.completed * int64(size)
|
||||
rps := 0.0
|
||||
kbps := 0.0
|
||||
if secs > 0 {
|
||||
rps = float64(r.completed) / secs
|
||||
kbps = float64(transferred) / 1024 / secs
|
||||
}
|
||||
fmt.Printf("\n%s results:\n", label)
|
||||
fmt.Printf("Concurrency Level: %d\n", concurrency)
|
||||
fmt.Printf("Time taken for tests: %.3f seconds\n", secs)
|
||||
fmt.Printf("Completed requests: %d\n", r.completed)
|
||||
fmt.Printf("Failed requests: %d\n", r.failed)
|
||||
fmt.Printf("Total transferred: %d bytes\n", transferred)
|
||||
fmt.Printf("Requests per second: %.2f [#/sec]\n", rps)
|
||||
fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", kbps)
|
||||
p := percentiles(r.latencies)
|
||||
fmt.Printf("Latency (ms): avg=%.2f p50=%.2f p90=%.2f p99=%.2f max=%.2f\n",
|
||||
ms(p.avg), ms(p.p50), ms(p.p90), ms(p.p99), ms(p.max))
|
||||
if r.failed > 0 && r.firstErr != nil {
|
||||
fmt.Printf("First error: %v\n", r.firstErr)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
type latencyStats struct {
|
||||
avg, p50, p90, p99, max time.Duration
|
||||
}
|
||||
|
||||
func percentiles(d []time.Duration) latencyStats {
|
||||
if len(d) == 0 {
|
||||
return latencyStats{}
|
||||
}
|
||||
slices.Sort(d)
|
||||
var sum time.Duration
|
||||
for _, v := range d {
|
||||
sum += v
|
||||
}
|
||||
at := func(q float64) time.Duration {
|
||||
idx := int(q * float64(len(d)))
|
||||
if idx >= len(d) {
|
||||
idx = len(d) - 1
|
||||
}
|
||||
return d[idx]
|
||||
}
|
||||
return latencyStats{
|
||||
avg: sum / time.Duration(len(d)),
|
||||
p50: at(0.50),
|
||||
p90: at(0.90),
|
||||
p99: at(0.99),
|
||||
max: d[len(d)-1],
|
||||
}
|
||||
}
|
||||
|
||||
func ms(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) }
|
||||
|
||||
func ensureBucket(client *s3.Client, bucket string) error {
|
||||
_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var owned *types.BucketAlreadyOwnedByYou
|
||||
var exists *types.BucketAlreadyExists
|
||||
if errors.As(err, &owned) || errors.As(err, &exists) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func fatalf(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
Reference in New Issue
Block a user