mirror of
https://github.com/versity/versitygw.git
synced 2026-08-01 04:46:14 +00:00
feat: Added gateway throughput performance test, got upload and download tests separated
This commit is contained in:
jonaustin09
+59
-1
@@ -17,6 +17,7 @@ var (
|
||||
objSize int64
|
||||
concurrency int
|
||||
files int
|
||||
totalReqs int
|
||||
upload bool
|
||||
download bool
|
||||
pathStyle bool
|
||||
@@ -175,7 +176,64 @@ func initTestCommands() []*cli.Command {
|
||||
|
||||
s3conf := integration.NewS3Conf(opts...)
|
||||
|
||||
return integration.TestPerformance(s3conf, upload, download, files, objSize, dstBucket, prefix)
|
||||
if upload {
|
||||
return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
|
||||
} else {
|
||||
return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "throughput",
|
||||
Usage: "Runs throughput performance test on the gateway",
|
||||
Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.IntFlag{
|
||||
Name: "reqs",
|
||||
Usage: "Total number of requests to send.",
|
||||
Value: 1000,
|
||||
Destination: &totalReqs,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "bucket",
|
||||
Usage: "Destination bucket name to make the requests",
|
||||
Destination: &dstBucket,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "concurrency",
|
||||
Usage: "threads per request",
|
||||
Value: 1,
|
||||
Destination: &concurrency,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "checksumDis",
|
||||
Usage: "Disable server checksum",
|
||||
Value: false,
|
||||
Destination: &checksumDisable,
|
||||
},
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
if dstBucket == "" {
|
||||
return fmt.Errorf("must specify the destination bucket")
|
||||
}
|
||||
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithConcurrency(concurrency),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
if checksumDisable {
|
||||
opts = append(opts, integration.WithDisableChecksum())
|
||||
}
|
||||
|
||||
s3conf := integration.NewS3Conf(opts...)
|
||||
|
||||
return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type prefResult struct {
|
||||
elapsed time.Duration
|
||||
size int64
|
||||
err error
|
||||
}
|
||||
|
||||
func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
|
||||
var sg sync.WaitGroup
|
||||
results := make([]prefResult, files)
|
||||
start := time.Now()
|
||||
if objSize == 0 {
|
||||
return fmt.Errorf("must specify object size for upload")
|
||||
}
|
||||
|
||||
if objSize > (int64(10000) * s.PartSize) {
|
||||
return fmt.Errorf("object size can not exceed 10000 * chunksize")
|
||||
}
|
||||
|
||||
runF("performance test: upload objects")
|
||||
|
||||
for i := 0; i < files; i++ {
|
||||
sg.Add(1)
|
||||
go func(i int) {
|
||||
var r io.Reader = NewDataReader(int(objSize), int(s.PartSize))
|
||||
|
||||
start := time.Now()
|
||||
err := s.UploadData(r, bucket, fmt.Sprintf("%v%v", prefix, i))
|
||||
results[i].elapsed = time.Since(start)
|
||||
results[i].err = err
|
||||
results[i].size = objSize
|
||||
sg.Done()
|
||||
}(i)
|
||||
}
|
||||
sg.Wait()
|
||||
elapsed := time.Since(start)
|
||||
|
||||
var tot int64
|
||||
for i, res := range results {
|
||||
if res.err != nil {
|
||||
failF("%v: %v\n", i, res.err)
|
||||
break
|
||||
}
|
||||
tot += res.size
|
||||
fmt.Printf("%v: %v in %v (%v MB/s)\n",
|
||||
i, res.size, res.elapsed,
|
||||
int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
passF("run upload: %v in %v (%v MB/s)\n",
|
||||
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
|
||||
var sg sync.WaitGroup
|
||||
results := make([]prefResult, files)
|
||||
start := time.Now()
|
||||
|
||||
runF("performance test: download objects")
|
||||
|
||||
for i := 0; i < files; i++ {
|
||||
sg.Add(1)
|
||||
go func(i int) {
|
||||
nw := NewNullWriter()
|
||||
start := time.Now()
|
||||
n, err := s.DownloadData(nw, bucket, fmt.Sprintf("%v%v", prefix, i))
|
||||
results[i].elapsed = time.Since(start)
|
||||
results[i].err = err
|
||||
results[i].size = n
|
||||
sg.Done()
|
||||
}(i)
|
||||
}
|
||||
sg.Wait()
|
||||
elapsed := time.Since(start)
|
||||
|
||||
var tot int64
|
||||
for i, res := range results {
|
||||
if res.err != nil {
|
||||
failF("%v: %v\n", i, res.err)
|
||||
break
|
||||
}
|
||||
tot += res.size
|
||||
fmt.Printf("%v: %v in %v (%v MB/s)\n",
|
||||
i, res.size, res.elapsed,
|
||||
int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
passF("run download: %v in %v (%v MB/s)\n",
|
||||
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
|
||||
client := s3.NewFromConfig(s.Config())
|
||||
var wg sync.WaitGroup
|
||||
var resErr error
|
||||
|
||||
// Record the start time
|
||||
startTime := time.Now()
|
||||
runF("performance test: measuring request per second")
|
||||
|
||||
for i := 0; i < s.Concurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < totalReqs/s.Concurrency; i++ {
|
||||
_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &bucket})
|
||||
if err != nil && resErr != nil {
|
||||
resErr = err
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
if resErr != nil {
|
||||
failF("performance test failed with error: %w", resErr)
|
||||
return nil
|
||||
}
|
||||
elapsedTime := time.Since(startTime)
|
||||
rps := int(float64(totalReqs) / elapsedTime.Seconds())
|
||||
|
||||
passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
|
||||
return nil
|
||||
}
|
||||
@@ -7,11 +7,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
@@ -3777,77 +3775,6 @@ func GetBucketAcl_success(s *S3Conf) {
|
||||
})
|
||||
}
|
||||
|
||||
type prefResult struct {
|
||||
elapsed time.Duration
|
||||
size int64
|
||||
err error
|
||||
}
|
||||
|
||||
func TestPerformance(s *S3Conf, upload, download bool, files int, objectSize int64, bucket, prefix string) error {
|
||||
var sg sync.WaitGroup
|
||||
results := make([]prefResult, files)
|
||||
start := time.Now()
|
||||
if upload {
|
||||
if objectSize == 0 {
|
||||
return fmt.Errorf("must specify object size for upload")
|
||||
}
|
||||
|
||||
if objectSize > (int64(10000) * s.PartSize) {
|
||||
return fmt.Errorf("object size can not exceed 10000 * chunksize")
|
||||
}
|
||||
|
||||
runF("performance test: upload/download objects")
|
||||
|
||||
for i := 0; i < files; i++ {
|
||||
sg.Add(1)
|
||||
go func(i int) {
|
||||
var r io.Reader = NewDataReader(int(objectSize), int(s.PartSize))
|
||||
|
||||
start := time.Now()
|
||||
err := s.UploadData(r, bucket, fmt.Sprintf("%v%v", prefix, i))
|
||||
results[i].elapsed = time.Since(start)
|
||||
results[i].err = err
|
||||
results[i].size = objectSize
|
||||
sg.Done()
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
if download {
|
||||
for i := 0; i < files; i++ {
|
||||
sg.Add(1)
|
||||
go func(i int) {
|
||||
nw := NewNullWriter()
|
||||
start := time.Now()
|
||||
n, err := s.DownloadData(nw, bucket, fmt.Sprintf("%v%v", prefix, i))
|
||||
results[i].elapsed = time.Since(start)
|
||||
results[i].err = err
|
||||
results[i].size = n
|
||||
sg.Done()
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
sg.Wait()
|
||||
elapsed := time.Since(start)
|
||||
|
||||
var tot int64
|
||||
for i, res := range results {
|
||||
if res.err != nil {
|
||||
failF("%v: %v\n", i, res.err)
|
||||
break
|
||||
}
|
||||
tot += res.size
|
||||
fmt.Printf("%v: %v in %v (%v MB/s)\n",
|
||||
i, res.size, res.elapsed,
|
||||
int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
passF("run perf: %v in %v (%v MB/s)\n",
|
||||
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Posix related tests
|
||||
func PutObject_overwrite_dir_obj(s *S3Conf) {
|
||||
testName := "PutObject_overwrite_dir_obj"
|
||||
|
||||
Reference in New Issue
Block a user