feat: Added s3 proxy comparison to upload, download and throughput benchmark tests

Throughput performance test

cmd/versitygw/test.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"fmt"
+	"math"
+	"os"
+	"text/tabwriter"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/integration"
@@ -13,10 +16,12 @@ var (
 	endpoint        string
 	prefix          string
 	dstBucket       string
+	proxyURL        string
 	partSize        int64
 	objSize         int64
 	concurrency     int
 	files           int
+	totalReqs       int
 	upload          bool
 	download        bool
 	pathStyle       bool
@@ -117,6 +122,7 @@ func initTestCommands() []*cli.Command {
 					Name:        "bucket",
 					Usage:       "Destination bucket name to read/write data",
 					Destination: &dstBucket,
+					Required:    true,
 				},
 				&cli.Int64Flag{
 					Name:        "partSize",
@@ -142,6 +148,11 @@ func initTestCommands() []*cli.Command {
 					Value:       false,
 					Destination: &checksumDisable,
 				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
 			},
 			Action: func(ctx *cli.Context) error {
 				if upload && download {
@@ -151,10 +162,6 @@ func initTestCommands() []*cli.Command {
 					return fmt.Errorf("must specify one of upload or download")
 				}
 
-				if dstBucket == "" {
-					return fmt.Errorf("must specify bucket")
-				}
-
 				opts := []integration.Option{
 					integration.WithAccess(awsID),
 					integration.WithSecret(awsSecret),
@@ -175,7 +182,123 @@ func initTestCommands() []*cli.Command {
 
 				s3conf := integration.NewS3Conf(opts...)
 
-				return integration.TestPerformance(s3conf, upload, download, files, objSize, dstBucket, prefix)
+				if upload {
+					if proxyURL == "" {
+						integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestUpload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 Server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 Proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
+				} else {
+					if proxyURL == "" {
+						integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestDownload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
+				}
+			},
+		},
+		{
+			Name:        "throughput",
+			Usage:       "Runs throughput performance test on the gateway",
+			Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
+			Flags: []cli.Flag{
+				&cli.IntFlag{
+					Name:        "reqs",
+					Usage:       "Total number of requests to send.",
+					Value:       1000,
+					Destination: &totalReqs,
+				},
+				&cli.StringFlag{
+					Name:        "bucket",
+					Usage:       "Destination bucket name to make the requests",
+					Destination: &dstBucket,
+				},
+				&cli.IntFlag{
+					Name:        "concurrency",
+					Usage:       "threads per request",
+					Value:       1,
+					Destination: &concurrency,
+				},
+				&cli.BoolFlag{
+					Name:        "checksumDis",
+					Usage:       "Disable server checksum",
+					Value:       false,
+					Destination: &checksumDisable,
+				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
+			},
+			Action: func(ctx *cli.Context) error {
+				opts := []integration.Option{
+					integration.WithAccess(awsID),
+					integration.WithSecret(awsSecret),
+					integration.WithRegion(region),
+					integration.WithEndpoint(endpoint),
+					integration.WithConcurrency(concurrency),
+				}
+				if debug {
+					opts = append(opts, integration.WithDebug())
+				}
+				if checksumDisable {
+					opts = append(opts, integration.WithDisableChecksum())
+				}
+
+				s3conf := integration.NewS3Conf(opts...)
+
+				if proxyURL == "" {
+					_, _, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					return err
+				} else {
+					elapsed, rps, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					opts = append(opts, integration.WithEndpoint(proxyURL))
+					s3proxy := integration.NewS3Conf(opts...)
+					proxyElapsed, proxyRPS, proxyErr := integration.TestReqPerSec(s3proxy, totalReqs, dstBucket)
+					if err != nil || proxyErr != nil {
+						return nil
+					}
+
+					printProxyResultsTable([][4]string{
+						{"    #    ", "Total Requests", "Time Taken", "Requests Per Second(Req/Sec)"},
+						{"---------", "--------------", "----------", "----------------------------"},
+						{"S3 Server", fmt.Sprint(totalReqs), fmt.Sprintf("%v", elapsed), fmt.Sprint(rps)},
+						{"S3 Proxy", fmt.Sprint(totalReqs), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(proxyRPS)},
+					})
+
+					return nil
+				}
 			},
 		},
 	}
@@ -206,3 +329,13 @@ func getAction(tf testFunc) func(*cli.Context) error {
 		return nil
 	}
 }
+
+func printProxyResultsTable(stats [][4]string) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	for _, elem := range stats {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", elem[0], elem[1], elem[2], elem[3])
+	}
+	fmt.Fprintln(w)
+	w.Flush()
+}

go.mod

@@ -3,31 +3,31 @@ module github.com/versity/versitygw
 go 1.20
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.21.2
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2
-	github.com/aws/smithy-go v1.15.0
+	github.com/aws/aws-sdk-go-v2 v1.22.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1
+	github.com/aws/smithy-go v1.16.0
 	github.com/go-ldap/ldap/v3 v3.4.6
 	github.com/gofiber/fiber/v2 v2.50.0
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.4.0
 	github.com/nats-io/nats.go v1.31.0
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.44
 	github.com/urfave/cli/v2 v2.25.7
 	github.com/valyala/fasthttp v1.50.0
 	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.13.0
+	golang.org/x/sys v0.14.0
 )
 
 require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/nats-io/nkeys v0.4.5 // indirect
+	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect
@@ -36,17 +36,17 @@ require (
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.19.0
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.43
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.24.0
+	github.com/aws/aws-sdk-go-v2/credentials v1.15.2
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -4,44 +4,44 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA=
-github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 h1:Sc82v7tDQ/vdU1WtuSyzZ1I7y/68j//HJ6uozND1IDs=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY=
-github.com/aws/aws-sdk-go-v2/config v1.19.0 h1:AdzDvwH6dWuVARCl3RTLGRc4Ogy+N7yLFxVxXe1ClQ0=
-github.com/aws/aws-sdk-go-v2/config v1.19.0/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91 h1:haAyxKHwoE+y/TJt+qHcPQf1dCViyyGbWcKjjYUllTE=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91/go.mod h1:ACQ6ta5YFlfSOz2c9A+EVYawLxFMZ0rI3Q0A0tGieKo=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 h1:wmGLw2i8ZTlHLw7a9ULGfQbuccw8uIiNr6sol5bFzc8=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6/go.mod h1:Q0Hq2X/NuL7z8b1Dww8rmOFl+jzusKEcyvkKspwdpyc=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 h1:7R8uRYyXzdD71KWVCL78lJZltah6VVznXBazvKjfH58=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15/go.mod h1:26SQUPcTNgV1Tapwdt4a1rOsYRsnBsJHLMPoxK2b0d8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 h1:skaFGzv+3kA+v2BPKhuekeb1Hbb105+44r8ASC+q5SE=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 h1:9ulSU5ClouoPIYhDQdg9tpl83d5Yb91PXTKK+17q+ow=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 h1:Ll5/YVCOzRB+gxPqs2uD0R7/MyATC0w85626glSKmp4=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2/go.mod h1:Zjfqt7KhQK+PO1bbOsFNzKgaq7TcxzmEoDWN8lM0qzQ=
-github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k=
-github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg=
-github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ=
-github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8=
-github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI=
+github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ=
+github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4=
+github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4=
+github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
+github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -54,9 +54,9 @@ github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8V
 github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
 github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -73,8 +73,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
 github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
-github.com/nats-io/nkeys v0.4.5 h1:Zdz2BUlFm4fJlierwvGK+yl20IAKUm7eV6AAZXEhkPk=
-github.com/nats-io/nkeys v0.4.5/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
+github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
+github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -145,8 +145,9 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

integration/bench.go

@@ -0,0 +1,142 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type prefResult struct {
+	elapsed time.Duration
+	size    int64
+	err     error
+}
+
+func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+	if objSize == 0 {
+		return 0, time.Since(start), fmt.Errorf("must specify object size for upload")
+	}
+
+	if objSize > (int64(10000) * s.PartSize) {
+		return 0, time.Since(start), fmt.Errorf("object size can not exceed 10000 * chunksize")
+	}
+
+	runF("performance test: upload objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			var r io.Reader = NewDataReader(int(objSize), int(s.PartSize))
+
+			start := time.Now()
+			err := s.UploadData(r, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = objSize
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed = time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			return 0, time.Since(start), res.err
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run upload: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return tot, time.Since(start), nil
+}
+
+func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
+	var sg sync.WaitGroup
+	results := make([]prefResult, files)
+	start := time.Now()
+
+	runF("performance test: download objects")
+
+	for i := 0; i < files; i++ {
+		sg.Add(1)
+		go func(i int) {
+			nw := NewNullWriter()
+			start := time.Now()
+			n, err := s.DownloadData(nw, bucket, fmt.Sprintf("%v%v", prefix, i))
+			results[i].elapsed = time.Since(start)
+			results[i].err = err
+			results[i].size = n
+			sg.Done()
+		}(i)
+	}
+	sg.Wait()
+	elapsed = time.Since(start)
+
+	var tot int64
+	for i, res := range results {
+		if res.err != nil {
+			failF("%v: %v\n", i, res.err)
+			return 0, elapsed, err
+		}
+		tot += res.size
+		fmt.Printf("%v: %v in %v (%v MB/s)\n",
+			i, res.size, res.elapsed,
+			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
+	}
+
+	fmt.Println()
+	passF("run download: %v in %v (%v MB/s)\n",
+		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
+
+	return tot, elapsed, nil
+}
+
+func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) (time.Duration, int, error) {
+	client := s3.NewFromConfig(s.Config())
+	var wg sync.WaitGroup
+	var resErr error
+
+	// Record the start time
+	startTime := time.Now()
+	runF("performance test: measuring request per second")
+
+	for i := 0; i < s.Concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < totalReqs/s.Concurrency; i++ {
+				_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &bucket})
+				if err != nil && resErr != nil {
+					resErr = err
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	if resErr != nil {
+		failF("performance test failed with error: %w", resErr)
+		return time.Since(startTime), 0, resErr
+	}
+	elapsedTime := time.Since(startTime)
+	rps := int(float64(totalReqs) / elapsedTime.Seconds())
+
+	passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
+	return elapsedTime, rps, nil
+}

integration/tests.go

@@ -7,11 +7,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
 	"regexp"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -3777,77 +3775,6 @@ func GetBucketAcl_success(s *S3Conf) {
 	})
 }
 
-type prefResult struct {
-	elapsed time.Duration
-	size    int64
-	err     error
-}
-
-func TestPerformance(s *S3Conf, upload, download bool, files int, objectSize int64, bucket, prefix string) error {
-	var sg sync.WaitGroup
-	results := make([]prefResult, files)
-	start := time.Now()
-	if upload {
-		if objectSize == 0 {
-			return fmt.Errorf("must specify object size for upload")
-		}
-
-		if objectSize > (int64(10000) * s.PartSize) {
-			return fmt.Errorf("object size can not exceed 10000 * chunksize")
-		}
-
-		runF("performance test: upload/download objects")
-
-		for i := 0; i < files; i++ {
-			sg.Add(1)
-			go func(i int) {
-				var r io.Reader = NewDataReader(int(objectSize), int(s.PartSize))
-
-				start := time.Now()
-				err := s.UploadData(r, bucket, fmt.Sprintf("%v%v", prefix, i))
-				results[i].elapsed = time.Since(start)
-				results[i].err = err
-				results[i].size = objectSize
-				sg.Done()
-			}(i)
-		}
-	}
-	if download {
-		for i := 0; i < files; i++ {
-			sg.Add(1)
-			go func(i int) {
-				nw := NewNullWriter()
-				start := time.Now()
-				n, err := s.DownloadData(nw, bucket, fmt.Sprintf("%v%v", prefix, i))
-				results[i].elapsed = time.Since(start)
-				results[i].err = err
-				results[i].size = n
-				sg.Done()
-			}(i)
-		}
-	}
-	sg.Wait()
-	elapsed := time.Since(start)
-
-	var tot int64
-	for i, res := range results {
-		if res.err != nil {
-			failF("%v: %v\n", i, res.err)
-			break
-		}
-		tot += res.size
-		fmt.Printf("%v: %v in %v (%v MB/s)\n",
-			i, res.size, res.elapsed,
-			int(math.Ceil(float64(res.size)/res.elapsed.Seconds())/1048576))
-	}
-
-	fmt.Println()
-	passF("run perf: %v in %v (%v MB/s)\n",
-		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
-
-	return nil
-}
-
 // Posix related tests
 func PutObject_overwrite_dir_obj(s *S3Conf) {
 	testName := "PutObject_overwrite_dir_obj"