diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000..8d492d9ca --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,286 @@ +name: "Performance" + +on: + push: + branches: [ master ] + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - '.github/workflows/performance.yml' + workflow_dispatch: + inputs: + profile_duration: + description: "CPU profiling duration in seconds" + required: false + default: "30" + type: string + benchmark_files: + description: "Number of files for the throughput benchmark" + required: false + default: "100000" + type: string + benchmark_concurrency: + description: "Concurrent read/write workers" + required: false + default: "16" + type: string + benchmark_size: + description: "Simulated file size in bytes" + required: false + default: "1024" + type: string + s3_objects: + description: "Number of objects for the S3 benchmark" + required: false + default: "20000" + type: string + s3_size: + description: "S3 object size in bytes" + required: false + default: "4096" + type: string + +concurrency: + group: ${{ github.head_ref || github.ref }}/performance + cancel-in-progress: true + +permissions: + contents: read + +jobs: + performance-profile: + name: CPU and Heap Profile + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Build weed + run: go build -o weed_bin ./weed + + - name: Start server with profiling enabled + run: | + mkdir -p ./perfdata + ./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \ + -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \ + -s3.port=8000 -s3.config=./docker/compose/s3.json \ + > weed.log 2>&1 & + echo "WEED_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then + echo "master is ready" + break + fi + sleep 1 + done + # give the volume server a moment to register with the master + sleep 3 + + - name: Capture profiles under load + run: | + DURATION="${{ github.event.inputs.profile_duration || '30' }}" + # drive write load so the sampled profile reflects real work + ./weed_bin benchmark -master=localhost:9333 -writeOnly \ + -c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 & + echo "Sampling CPU profile for ${DURATION}s..." + curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof + curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof + curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt + go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true + go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true + + - name: Profile summary + if: always() + run: | + { + echo "## CPU profile (top functions)" + echo '```' + head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Stop server + if: always() + run: kill "${WEED_PID}" 2>/dev/null || true + + - name: Show server log on failure + if: failure() + run: tail -200 weed.log || true + + - name: Upload profiles + if: always() + uses: actions/upload-artifact@v7 + with: + name: performance-profile-${{ github.run_number }} + path: | + cpu.pprof + heap.pprof + cpu-top.txt + heap-top.txt + goroutine.txt + weed.log + retention-days: 30 + + benchmark: + name: Throughput Benchmark + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Build weed + run: go build -o weed_bin ./weed + + - name: Start server + run: | + mkdir -p ./perfdata + ./weed_bin -v=1 server -dir=./perfdata -volume.max=0 \ + -master.volumeSizeLimitMB=1024 > weed.log 2>&1 & + echo "WEED_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then + echo "master is ready" + break + fi + sleep 1 + done + sleep 3 + + - name: Run throughput benchmark + run: | + N="${{ github.event.inputs.benchmark_files || '100000' }}" + C="${{ github.event.inputs.benchmark_concurrency || '16' }}" + SIZE="${{ github.event.inputs.benchmark_size || '1024' }}" + ./weed_bin benchmark -master=localhost:9333 \ + -c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt + + - name: Run Go micro-benchmarks + continue-on-error: true + run: | + go test -run='^$' -bench=. -benchmem -benchtime=10x \ + ./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \ + 2>&1 | tee go-benchmarks.txt + + - name: Benchmark summary + if: always() + run: | + { + echo "## Throughput benchmark" + echo '```' + grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \ + benchmark-results.txt 2>/dev/null || echo "no benchmark results captured" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Stop server + if: always() + run: kill "${WEED_PID}" 2>/dev/null || true + + - name: Show server log on failure + if: failure() + run: tail -200 weed.log || true + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-results-${{ github.run_number }} + path: | + benchmark-results.txt + go-benchmarks.txt + weed.log + retention-days: 7 + + s3-benchmark: + name: S3 Read/Write Benchmark + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Build weed and S3 load tool + run: | + go build -o weed_bin ./weed + go build -o s3bench ./test/s3/benchmark + + - name: Start server with S3 gateway + run: | + mkdir -p ./perfdata + ./weed_bin -v=1 server -dir=./perfdata -s3 -filer \ + -volume.max=0 -master.volumeSizeLimitMB=1024 \ + -s3.port=8000 -s3.config=./docker/compose/s3.json \ + > weed.log 2>&1 & + echo "WEED_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then + echo "master is ready" + break + fi + sleep 1 + done + for i in $(seq 1 30); do + if nc -z localhost 8000 2>/dev/null; then + echo "s3 gateway is ready" + break + fi + sleep 1 + done + sleep 2 + + - name: Run S3 read/write benchmark + run: | + OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}" + C="${{ github.event.inputs.benchmark_concurrency || '16' }}" + SIZE="${{ github.event.inputs.s3_size || '4096' }}" + ./s3bench -endpoint=http://localhost:8000 \ + -access-key=some_access_key1 -secret-key=some_secret_key1 \ + -objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \ + 2>&1 | tee s3-benchmark-results.txt + + - name: S3 benchmark summary + if: always() + run: | + { + echo "## S3 read/write benchmark" + echo '```' + grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \ + s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Stop server + if: always() + run: kill "${WEED_PID}" 2>/dev/null || true + + - name: Show server log on failure + if: failure() + run: tail -200 weed.log || true + + - name: Upload S3 benchmark results + if: always() + uses: actions/upload-artifact@v7 + with: + name: s3-benchmark-results-${{ github.run_number }} + path: | + s3-benchmark-results.txt + weed.log + retention-days: 7 diff --git a/test/s3/benchmark/s3_benchmark.go b/test/s3/benchmark/s3_benchmark.go new file mode 100644 index 000000000..2f104fe00 --- /dev/null +++ b/test/s3/benchmark/s3_benchmark.go @@ -0,0 +1,223 @@ +// Command s3_benchmark drives concurrent PUT and GET load against an S3 +// gateway and reports throughput and latency. It is a self-contained load +// generator for the performance CI, using the aws-sdk-go-v2 client the rest +// of the S3 tests already depend on. +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +func main() { + endpoint := flag.String("endpoint", "http://localhost:8000", "S3 gateway endpoint") + accessKey := flag.String("access-key", "some_access_key1", "S3 access key") + secretKey := flag.String("secret-key", "some_secret_key1", "S3 secret key") + region := flag.String("region", "us-east-1", "S3 region") + bucket := flag.String("bucket", "perf-benchmark", "bucket to write into") + objects := flag.Int("objects", 10000, "number of objects") + size := flag.Int("size", 1024, "object size in bytes") + concurrency := flag.Int("concurrency", 16, "concurrent workers") + mode := flag.String("mode", "both", "write, read, or both") + flag.Parse() + + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(*region), + config.WithRetryMaxAttempts(1), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(*accessKey, *secretKey, "")), + ) + if err != nil { + fatalf("load aws config: %v", err) + } + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(*endpoint) + o.UsePathStyle = true + }) + + if err := ensureBucket(client, *bucket); err != nil { + fatalf("ensure bucket: %v", err) + } + + payload := make([]byte, *size) + for i := range payload { + payload[i] = byte(i) + } + + doWrite := *mode == "both" || *mode == "write" + doRead := *mode == "both" || *mode == "read" + + if doWrite { + report("S3 WRITE", *concurrency, *size, run(*concurrency, *objects, func(key string) error { + _, err := client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: bucket, + Key: aws.String(key), + Body: bytes.NewReader(payload), + ContentLength: aws.Int64(int64(*size)), + }) + return err + })) + } + + if doRead { + report("S3 READ", *concurrency, *size, run(*concurrency, *objects, func(key string) error { + out, err := client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: bucket, + Key: aws.String(key), + }) + if err != nil { + return err + } + defer out.Body.Close() + _, err = io.Copy(io.Discard, out.Body) + return err + })) + } +} + +type result struct { + completed int64 + failed int64 + elapsed time.Duration + latencies []time.Duration + firstErr error +} + +// run fans `total` keyed operations across `workers` goroutines, timing each. +func run(workers, total int, op func(key string) error) result { + var ( + completed, failed, nextKey atomic.Int64 + mu sync.Mutex + firstErr error + latencies = make([]time.Duration, 0, total) + wg sync.WaitGroup + ) + start := time.Now() + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + local := make([]time.Duration, 0, total/workers+1) + for { + i := nextKey.Add(1) - 1 + if i >= int64(total) { + break + } + key := fmt.Sprintf("obj-%d", i) + opStart := time.Now() + err := op(key) + local = append(local, time.Since(opStart)) + if err != nil { + failed.Add(1) + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + continue + } + completed.Add(1) + } + mu.Lock() + latencies = append(latencies, local...) + mu.Unlock() + }() + } + wg.Wait() + return result{ + completed: completed.Load(), + failed: failed.Load(), + elapsed: time.Since(start), + latencies: latencies, + firstErr: firstErr, + } +} + +func report(label string, concurrency, size int, r result) { + secs := r.elapsed.Seconds() + transferred := r.completed * int64(size) + rps := 0.0 + kbps := 0.0 + if secs > 0 { + rps = float64(r.completed) / secs + kbps = float64(transferred) / 1024 / secs + } + fmt.Printf("\n%s results:\n", label) + fmt.Printf("Concurrency Level: %d\n", concurrency) + fmt.Printf("Time taken for tests: %.3f seconds\n", secs) + fmt.Printf("Completed requests: %d\n", r.completed) + fmt.Printf("Failed requests: %d\n", r.failed) + fmt.Printf("Total transferred: %d bytes\n", transferred) + fmt.Printf("Requests per second: %.2f [#/sec]\n", rps) + fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", kbps) + p := percentiles(r.latencies) + fmt.Printf("Latency (ms): avg=%.2f p50=%.2f p90=%.2f p99=%.2f max=%.2f\n", + ms(p.avg), ms(p.p50), ms(p.p90), ms(p.p99), ms(p.max)) + if r.failed > 0 && r.firstErr != nil { + fmt.Printf("First error: %v\n", r.firstErr) + os.Exit(1) + } +} + +type latencyStats struct { + avg, p50, p90, p99, max time.Duration +} + +func percentiles(d []time.Duration) latencyStats { + if len(d) == 0 { + return latencyStats{} + } + slices.Sort(d) + var sum time.Duration + for _, v := range d { + sum += v + } + at := func(q float64) time.Duration { + idx := int(q * float64(len(d))) + if idx >= len(d) { + idx = len(d) - 1 + } + return d[idx] + } + return latencyStats{ + avg: sum / time.Duration(len(d)), + p50: at(0.50), + p90: at(0.90), + p99: at(0.99), + max: d[len(d)-1], + } +} + +func ms(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) } + +func ensureBucket(client *s3.Client, bucket string) error { + _, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{Bucket: aws.String(bucket)}) + if err == nil { + return nil + } + var owned *types.BucketAlreadyOwnedByYou + var exists *types.BucketAlreadyExists + if errors.As(err, &owned) || errors.As(err, &exists) { + return nil + } + return err +} + +func fatalf(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +}