feat: test CLI command set up for client side testing, test cases are corresponded with subcommands, added full-flow test case

cmd/versitygw/main.go

@@ -53,6 +53,7 @@ func main() {
 		posixCommand(),
 		scoutfsCommand(),
 		adminCommand(),
+		testCommand(),
 	}
 
 	if err := app.Run(os.Args); err != nil {

cmd/versitygw/test.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/integration"
+)
+
+var (
+	awsID     string
+	awsSecret string
+	endpoint  string
+)
+
+func testCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "test",
+		Usage: "Client side testing command for the gateway",
+		Description: `The testing CLI is used to test group of versitygw actions.
+		It also includes some performance and stress testing`,
+		Subcommands: initTestCommands(),
+		Flags:       initTestFlags(),
+	}
+}
+
+func initTestFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "access",
+			Usage:       "aws user access key",
+			EnvVars:     []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY"},
+			Aliases:     []string{"a"},
+			Destination: &awsID,
+		},
+		&cli.StringFlag{
+			Name:        "secret",
+			Usage:       "aws user secret access key",
+			EnvVars:     []string{"AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY"},
+			Aliases:     []string{"s"},
+			Destination: &awsSecret,
+		},
+		&cli.StringFlag{
+			Name:        "endpoint",
+			Usage:       "s3 server endpoint",
+			Destination: &endpoint,
+			Aliases:     []string{"e"},
+		},
+		&cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "enable debug mode",
+			Aliases:     []string{"d"},
+			Destination: &debug,
+		},
+	}
+}
+
+func initTestCommands() []*cli.Command {
+	return []*cli.Command{
+		{
+			Name:  "make-bucket",
+			Usage: "Test bucket creation.",
+			Description: `Calls s3 gateway create-bucket action to create a new bucket,
+		then calls delete-bucket action to delete the bucket.`,
+			Action: getAction(integration.TestMakeBucket),
+		},
+		{
+			Name:  "put-get-object",
+			Usage: "Test put & get object.",
+			Description: `Creates a bucket with s3 gateway action, puts an object in it,
+			gets the object from the bucket, deletes both the object and bucket.`,
+			Action: getAction(integration.TestPutGetObject),
+		},
+		{
+			Name:  "put-get-mp-object",
+			Usage: "Test put & get multipart object.",
+			Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload,
+			gets the object from the bucket, deletes both the object and bucket.`,
+			Action: getAction(integration.TestPutGetMPObject),
+		},
+		{
+			Name:  "put-dir-object",
+			Usage: "Test put directory object.",
+			Description: `Creates a bucket with s3 gateway action, puts a directory object in it,
+			lists the bucket's objects, deletes both the objects and bucket.`,
+			Action: getAction(integration.TestPutDirObject),
+		},
+		{
+			Name:  "list-objects",
+			Usage: "Test list-objects action.",
+			Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it,
+			lists the bucket's objects, deletes both the objects and bucket.`,
+			Action: getAction(integration.TestListObject),
+		},
+		{
+			Name:  "abort-mp",
+			Usage: "Tests abort-multipart-upload action.",
+			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
+			lists the multipart upload, aborts the multipart upload, lists the multipart upload again,
+			deletes both the objects and bucket.`,
+			Action: getAction(integration.TestListAbortMultiPartObject),
+		},
+		{
+			Name:  "list-parts",
+			Usage: "Tests list-parts action.",
+			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
+			lists the upload parts, deletes both the objects and bucket.`,
+			Action: getAction(integration.TestListMultiParts),
+		},
+		{
+			Name:  "incorrect-mp",
+			Usage: "Tests incorrect multipart case.",
+			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
+			uploads different parts, completes the multipart upload with incorrect part numbers,
+			calls the head-object action, compares the content length, removes both the object and bucket`,
+			Action: getAction(integration.TestIncorrectMultiParts),
+		},
+		{
+			Name:  "incomplete-mp",
+			Usage: "Tests incomplete multi parts.",
+			Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
+			upload a part, lists the parts, checks if the uploaded part is in the list, 
+			removes both the object and the bucket`,
+			Action: getAction(integration.TestIncompleteMultiParts),
+		},
+		{
+			Name:  "incomplete-put-object",
+			Usage: "Tests incomplete put objects case.",
+			Description: `Creates a bucket with s3 gateway action, puts an object in it,
+			gets the object with head-object action, expects the object to be got, 
+			removes both the object and bucket`,
+			Action: getAction(integration.TestIncompletePutObject),
+		},
+		{
+			Name:  "get-range",
+			Usage: "Tests get object by range.",
+			Description: `Creates a bucket with s3 gateway action, puts an object in it,
+			gets the object by specifying the object range, compares the range with the original one,
+			removes both the object and the bucket`,
+			Action: getAction(integration.TestRangeGet),
+		},
+		{
+			Name:  "invalid-mp",
+			Usage: "Tests invalid multi part case.",
+			Description: `Creates a bucket with s3 gateway action, creates a multi part upload,
+			uploads an invalid part, gets the object with head-object action, expects to get error,
+			removes both the object and bucket`,
+			Action: getAction(integration.TestInvalidMultiParts),
+		},
+		{
+			Name:        "full-flow",
+			Usage:       "Tests the full flow of gateway.",
+			Description: `Runs all the available tests to test the full flow of the gateway.`,
+			Action:      getAction(integration.TestFullFlow),
+		},
+	}
+}
+
+type testFunc func(*integration.S3Conf)
+
+func getAction(tf testFunc) func(*cli.Context) error {
+	return func(ctx *cli.Context) error {
+		opts := []integration.Option{
+			integration.WithAccess(awsID),
+			integration.WithSecret(awsSecret),
+			integration.WithRegion(region),
+			integration.WithEndpoint(endpoint),
+		}
+		if debug {
+			opts = append(opts, integration.WithDebug())
+		}
+
+		s := integration.NewS3Conf(opts...)
+		tf(s)
+
+		fmt.Println()
+		fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount)
+		return nil
+	}
+}

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/aws/aws-sdk-go-v2 v1.18.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0
 	github.com/aws/smithy-go v1.13.5
 	github.com/gofiber/fiber/v2 v2.46.0
 	github.com/google/uuid v1.3.0
@@ -15,9 +15,21 @@ require (
 	golang.org/x/sys v0.9.0
 )
 
+require (
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+)
+
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.27
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.26
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect

go.sum

@@ -4,10 +4,20 @@ github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCe
 github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
+github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA=
+github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70 h1:4bh28MeeXoBFTjb0JjQ5sVatzlf5xA1DziV8mZed9v4=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70/go.mod h1:9yI5NXzqy2yOiMytv6QLZHvlyHLwYxO9iIq+bZIbrFg=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
@@ -20,6 +30,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJ
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1 h1:rYYwwsGqbwvGgQHjBkqgDt8MynXk+I8xgS0IEj5gOT0=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 h1:ya7fmrN2fE7s1P2gaPbNg5MTkERVWfsH8ToP1YC4Z9o=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY=
+github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg=
 github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
@@ -31,6 +49,7 @@ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=

integration/output.go

@@ -0,0 +1,31 @@
+package integration
+
+import "fmt"
+
+var (
+	colorReset = "\033[0m"
+	colorRed   = "\033[31m"
+	colorGreen = "\033[32m"
+	colorCyan  = "\033[36m"
+)
+
+var (
+	RunCount  = 0
+	PassCount = 0
+	FailCount = 0
+)
+
+func runF(format string, a ...interface{}) {
+	RunCount++
+	fmt.Printf(colorCyan+"RUN  "+colorReset+format+"\n", a...)
+}
+
+func failF(format string, a ...interface{}) {
+	FailCount++
+	fmt.Printf(colorRed+"FAIL "+colorReset+format+"\n", a...)
+}
+
+func passF(format string, a ...interface{}) {
+	PassCount++
+	fmt.Printf(colorGreen+"PASS "+colorReset+format+"\n", a...)
+}

integration/reader.go

@@ -0,0 +1,54 @@
+package integration
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"hash"
+	"io"
+)
+
+type RReader struct {
+	buf      []byte
+	dataleft int
+	hash     hash.Hash
+}
+
+func NewDataReader(totalsize, bufsize int) *RReader {
+	b := make([]byte, bufsize)
+	rand.Read(b)
+	return &RReader{
+		buf:      b,
+		dataleft: totalsize,
+		hash:     sha256.New(),
+	}
+}
+
+func (r *RReader) Read(p []byte) (int, error) {
+	n := min(len(p), len(r.buf), r.dataleft)
+	r.dataleft -= n
+	err := error(nil)
+	if n == 0 {
+		err = io.EOF
+	}
+	r.hash.Write(r.buf[:n])
+	return copy(p, r.buf[:n]), err
+}
+
+func (r *RReader) Sum() []byte {
+	return r.hash.Sum(nil)
+}
+
+func min(values ...int) int {
+	if len(values) == 0 {
+		return 0
+	}
+
+	min := values[0]
+	for _, v := range values {
+		if v < min {
+			min = v
+		}
+	}
+
+	return min
+}

integration/s3conf.go

@@ -0,0 +1,128 @@
+package integration
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/smithy-go/middleware"
+)
+
+type S3Conf struct {
+	awsID           string
+	awsSecret       string
+	awsRegion       string
+	endpoint        string
+	checksumDisable bool
+	pathStyle       bool
+	PartSize        int64
+	Concurrency     int
+	debug           bool
+}
+
+func NewS3Conf(opts ...Option) *S3Conf {
+	s := &S3Conf{
+		PartSize:    64 * 1024 * 1024, // 64B default chunksize
+		Concurrency: 1,                // 1 default concurrency
+	}
+
+	for _, opt := range opts {
+		opt(s)
+	}
+	return s
+}
+
+type Option func(*S3Conf)
+
+func WithAccess(ak string) Option {
+	return func(s *S3Conf) { s.awsID = ak }
+}
+func WithSecret(sk string) Option {
+	return func(s *S3Conf) { s.awsSecret = sk }
+}
+func WithRegion(r string) Option {
+	return func(s *S3Conf) { s.awsRegion = r }
+}
+func WithEndpoint(e string) Option {
+	return func(s *S3Conf) { s.endpoint = e }
+}
+func WithDisableChecksum() Option {
+	return func(s *S3Conf) { s.checksumDisable = true }
+}
+func WithPathStyle() Option {
+	return func(s *S3Conf) { s.pathStyle = true }
+}
+func WithPartSize(p int64) Option {
+	return func(s *S3Conf) { s.PartSize = p }
+}
+func WithConcurrency(c int) Option {
+	return func(s *S3Conf) { s.Concurrency = c }
+}
+func WithDebug() Option {
+	return func(s *S3Conf) { s.debug = true }
+}
+
+func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
+	// TODO support token/IAM
+	if c.awsSecret == "" {
+		c.awsSecret = os.Getenv("AWS_SECRET_ACCESS_KEY")
+	}
+	if c.awsSecret == "" {
+		log.Fatal("no AWS_SECRET_ACCESS_KEY found")
+	}
+
+	return credentials.NewStaticCredentialsProvider(c.awsID, c.awsSecret, "")
+}
+
+func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               c.endpoint,
+		SigningRegion:     c.awsRegion,
+		HostnameImmutable: true,
+	}, nil
+}
+
+func (c *S3Conf) Config() aws.Config {
+	creds := c.getCreds()
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(c.awsRegion),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if c.endpoint != "" && c.endpoint != "aws" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(c))
+	}
+
+	if c.checksumDisable {
+		opts = append(opts,
+			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
+	}
+
+	if c.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	cfg, err := config.LoadDefaultConfig(
+		context.TODO(), opts...)
+	if err != nil {
+		log.Fatalln("error:", err)
+	}
+
+	return cfg
+}