From 24ae7a2e86e5eaa31de8e2178de26b1dc4ab8d9e Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Thu, 22 Jun 2023 19:49:30 +0400 Subject: [PATCH 1/2] feat: test CLI command set up for client side testing, test cases are corresponded with subcommands, added full-flow test case --- cmd/versitygw/main.go | 1 + cmd/versitygw/test.go | 180 +++++++ go.mod | 14 +- go.sum | 19 + integration/output.go | 31 ++ integration/reader.go | 54 ++ integration/s3conf.go | 128 +++++ integration/tests.go | 1166 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 1592 insertions(+), 1 deletion(-) create mode 100644 cmd/versitygw/test.go create mode 100644 integration/output.go create mode 100644 integration/reader.go create mode 100644 integration/s3conf.go create mode 100644 integration/tests.go diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index dd797a9c..b307ffa9 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -53,6 +53,7 @@ func main() { posixCommand(), scoutfsCommand(), adminCommand(), + testCommand(), } if err := app.Run(os.Args); err != nil { diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go new file mode 100644 index 00000000..d12418fc --- /dev/null +++ b/cmd/versitygw/test.go @@ -0,0 +1,180 @@ +package main + +import ( + "fmt" + + "github.com/urfave/cli/v2" + "github.com/versity/versitygw/integration" +) + +var ( + awsID string + awsSecret string + endpoint string +) + +func testCommand() *cli.Command { + return &cli.Command{ + Name: "test", + Usage: "Client side testing command for the gateway", + Description: `The testing CLI is used to test group of versitygw actions. + It also includes some performance and stress testing`, + Subcommands: initTestCommands(), + Flags: initTestFlags(), + } +} + +func initTestFlags() []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "access", + Usage: "aws user access key", + EnvVars: []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY"}, + Aliases: []string{"a"}, + Destination: &awsID, + }, + &cli.StringFlag{ + Name: "secret", + Usage: "aws user secret access key", + EnvVars: []string{"AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY"}, + Aliases: []string{"s"}, + Destination: &awsSecret, + }, + &cli.StringFlag{ + Name: "endpoint", + Usage: "s3 server endpoint", + Destination: &endpoint, + Aliases: []string{"e"}, + }, + &cli.BoolFlag{ + Name: "debug", + Usage: "enable debug mode", + Aliases: []string{"d"}, + Destination: &debug, + }, + } +} + +func initTestCommands() []*cli.Command { + return []*cli.Command{ + { + Name: "make-bucket", + Usage: "Test bucket creation.", + Description: `Calls s3 gateway create-bucket action to create a new bucket, + then calls delete-bucket action to delete the bucket.`, + Action: getAction(integration.TestMakeBucket), + }, + { + Name: "put-get-object", + Usage: "Test put & get object.", + Description: `Creates a bucket with s3 gateway action, puts an object in it, + gets the object from the bucket, deletes both the object and bucket.`, + Action: getAction(integration.TestPutGetObject), + }, + { + Name: "put-get-mp-object", + Usage: "Test put & get multipart object.", + Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload, + gets the object from the bucket, deletes both the object and bucket.`, + Action: getAction(integration.TestPutGetMPObject), + }, + { + Name: "put-dir-object", + Usage: "Test put directory object.", + Description: `Creates a bucket with s3 gateway action, puts a directory object in it, + lists the bucket's objects, deletes both the objects and bucket.`, + Action: getAction(integration.TestPutDirObject), + }, + { + Name: "list-objects", + Usage: "Test list-objects action.", + Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it, + lists the bucket's objects, deletes both the objects and bucket.`, + Action: getAction(integration.TestListObject), + }, + { + Name: "abort-mp", + Usage: "Tests abort-multipart-upload action.", + Description: `Creates a bucket with s3 gateway action, creates a multipart upload, + lists the multipart upload, aborts the multipart upload, lists the multipart upload again, + deletes both the objects and bucket.`, + Action: getAction(integration.TestListAbortMultiPartObject), + }, + { + Name: "list-parts", + Usage: "Tests list-parts action.", + Description: `Creates a bucket with s3 gateway action, creates a multipart upload, + lists the upload parts, deletes both the objects and bucket.`, + Action: getAction(integration.TestListMultiParts), + }, + { + Name: "incorrect-mp", + Usage: "Tests incorrect multipart case.", + Description: `Creates a bucket with s3 gateway action, creates a multipart upload, + uploads different parts, completes the multipart upload with incorrect part numbers, + calls the head-object action, compares the content length, removes both the object and bucket`, + Action: getAction(integration.TestIncorrectMultiParts), + }, + { + Name: "incomplete-mp", + Usage: "Tests incomplete multi parts.", + Description: `Creates a bucket with s3 gateway action, creates a multipart upload, + upload a part, lists the parts, checks if the uploaded part is in the list, + removes both the object and the bucket`, + Action: getAction(integration.TestIncompleteMultiParts), + }, + { + Name: "incomplete-put-object", + Usage: "Tests incomplete put objects case.", + Description: `Creates a bucket with s3 gateway action, puts an object in it, + gets the object with head-object action, expects the object to be got, + removes both the object and bucket`, + Action: getAction(integration.TestIncompletePutObject), + }, + { + Name: "get-range", + Usage: "Tests get object by range.", + Description: `Creates a bucket with s3 gateway action, puts an object in it, + gets the object by specifying the object range, compares the range with the original one, + removes both the object and the bucket`, + Action: getAction(integration.TestRangeGet), + }, + { + Name: "invalid-mp", + Usage: "Tests invalid multi part case.", + Description: `Creates a bucket with s3 gateway action, creates a multi part upload, + uploads an invalid part, gets the object with head-object action, expects to get error, + removes both the object and bucket`, + Action: getAction(integration.TestInvalidMultiParts), + }, + { + Name: "full-flow", + Usage: "Tests the full flow of gateway.", + Description: `Runs all the available tests to test the full flow of the gateway.`, + Action: getAction(integration.TestFullFlow), + }, + } +} + +type testFunc func(*integration.S3Conf) + +func getAction(tf testFunc) func(*cli.Context) error { + return func(ctx *cli.Context) error { + opts := []integration.Option{ + integration.WithAccess(awsID), + integration.WithSecret(awsSecret), + integration.WithRegion(region), + integration.WithEndpoint(endpoint), + } + if debug { + opts = append(opts, integration.WithDebug()) + } + + s := integration.NewS3Conf(opts...) + tf(s) + + fmt.Println() + fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount) + return nil + } +} diff --git a/go.mod b/go.mod index 281f9b5a..18b2893f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/aws/aws-sdk-go-v2 v1.18.1 - github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 github.com/aws/smithy-go v1.13.5 github.com/gofiber/fiber/v2 v2.46.0 github.com/google/uuid v1.3.0 @@ -15,9 +15,21 @@ require ( golang.org/x/sys v0.9.0 ) +require ( + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect +) + require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.27 + github.com/aws/aws-sdk-go-v2/credentials v1.13.26 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect diff --git a/go.sum b/go.sum index cdbc00a8..7d33344e 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,20 @@ github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCe github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= +github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= +github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= +github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70 h1:4bh28MeeXoBFTjb0JjQ5sVatzlf5xA1DziV8mZed9v4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70/go.mod h1:9yI5NXzqy2yOiMytv6QLZHvlyHLwYxO9iIq+bZIbrFg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= @@ -20,6 +30,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1 h1:rYYwwsGqbwvGgQHjBkqgDt8MynXk+I8xgS0IEj5gOT0= github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 h1:ya7fmrN2fE7s1P2gaPbNg5MTkERVWfsH8ToP1YC4Z9o= +github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -31,6 +49,7 @@ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= diff --git a/integration/output.go b/integration/output.go new file mode 100644 index 00000000..e4ad1479 --- /dev/null +++ b/integration/output.go @@ -0,0 +1,31 @@ +package integration + +import "fmt" + +var ( + colorReset = "\033[0m" + colorRed = "\033[31m" + colorGreen = "\033[32m" + colorCyan = "\033[36m" +) + +var ( + RunCount = 0 + PassCount = 0 + FailCount = 0 +) + +func runF(format string, a ...interface{}) { + RunCount++ + fmt.Printf(colorCyan+"RUN "+colorReset+format+"\n", a...) +} + +func failF(format string, a ...interface{}) { + FailCount++ + fmt.Printf(colorRed+"FAIL "+colorReset+format+"\n", a...) +} + +func passF(format string, a ...interface{}) { + PassCount++ + fmt.Printf(colorGreen+"PASS "+colorReset+format+"\n", a...) +} diff --git a/integration/reader.go b/integration/reader.go new file mode 100644 index 00000000..7a8d2d27 --- /dev/null +++ b/integration/reader.go @@ -0,0 +1,54 @@ +package integration + +import ( + "crypto/rand" + "crypto/sha256" + "hash" + "io" +) + +type RReader struct { + buf []byte + dataleft int + hash hash.Hash +} + +func NewDataReader(totalsize, bufsize int) *RReader { + b := make([]byte, bufsize) + rand.Read(b) + return &RReader{ + buf: b, + dataleft: totalsize, + hash: sha256.New(), + } +} + +func (r *RReader) Read(p []byte) (int, error) { + n := min(len(p), len(r.buf), r.dataleft) + r.dataleft -= n + err := error(nil) + if n == 0 { + err = io.EOF + } + r.hash.Write(r.buf[:n]) + return copy(p, r.buf[:n]), err +} + +func (r *RReader) Sum() []byte { + return r.hash.Sum(nil) +} + +func min(values ...int) int { + if len(values) == 0 { + return 0 + } + + min := values[0] + for _, v := range values { + if v < min { + min = v + } + } + + return min +} diff --git a/integration/s3conf.go b/integration/s3conf.go new file mode 100644 index 00000000..4bb02a8c --- /dev/null +++ b/integration/s3conf.go @@ -0,0 +1,128 @@ +package integration + +import ( + "context" + "crypto/tls" + "log" + "net/http" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/smithy-go/middleware" +) + +type S3Conf struct { + awsID string + awsSecret string + awsRegion string + endpoint string + checksumDisable bool + pathStyle bool + PartSize int64 + Concurrency int + debug bool +} + +func NewS3Conf(opts ...Option) *S3Conf { + s := &S3Conf{ + PartSize: 64 * 1024 * 1024, // 64B default chunksize + Concurrency: 1, // 1 default concurrency + } + + for _, opt := range opts { + opt(s) + } + return s +} + +type Option func(*S3Conf) + +func WithAccess(ak string) Option { + return func(s *S3Conf) { s.awsID = ak } +} +func WithSecret(sk string) Option { + return func(s *S3Conf) { s.awsSecret = sk } +} +func WithRegion(r string) Option { + return func(s *S3Conf) { s.awsRegion = r } +} +func WithEndpoint(e string) Option { + return func(s *S3Conf) { s.endpoint = e } +} +func WithDisableChecksum() Option { + return func(s *S3Conf) { s.checksumDisable = true } +} +func WithPathStyle() Option { + return func(s *S3Conf) { s.pathStyle = true } +} +func WithPartSize(p int64) Option { + return func(s *S3Conf) { s.PartSize = p } +} +func WithConcurrency(c int) Option { + return func(s *S3Conf) { s.Concurrency = c } +} +func WithDebug() Option { + return func(s *S3Conf) { s.debug = true } +} + +func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider { + // TODO support token/IAM + if c.awsSecret == "" { + c.awsSecret = os.Getenv("AWS_SECRET_ACCESS_KEY") + } + if c.awsSecret == "" { + log.Fatal("no AWS_SECRET_ACCESS_KEY found") + } + + return credentials.NewStaticCredentialsProvider(c.awsID, c.awsSecret, "") +} + +func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: c.endpoint, + SigningRegion: c.awsRegion, + HostnameImmutable: true, + }, nil +} + +func (c *S3Conf) Config() aws.Config { + creds := c.getCreds() + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + opts := []func(*config.LoadOptions) error{ + config.WithRegion(c.awsRegion), + config.WithCredentialsProvider(creds), + config.WithHTTPClient(client), + } + + if c.endpoint != "" && c.endpoint != "aws" { + opts = append(opts, + config.WithEndpointResolverWithOptions(c)) + } + + if c.checksumDisable { + opts = append(opts, + config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware})) + } + + if c.debug { + opts = append(opts, + config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage)) + } + + cfg, err := config.LoadDefaultConfig( + context.TODO(), opts...) + if err != nil { + log.Fatalln("error:", err) + } + + return cfg +} diff --git a/integration/tests.go b/integration/tests.go new file mode 100644 index 00000000..f981a3a6 --- /dev/null +++ b/integration/tests.go @@ -0,0 +1,1166 @@ +package integration + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +var ( + shortTimeout = 10 * time.Second +) + +func setup(s *S3Conf, bucket string) error { + s3client := s3.NewFromConfig(s.Config()) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: &bucket, + }) + cancel() + return err +} + +func teardown(s *S3Conf, bucket string) error { + s3client := s3.NewFromConfig(s.Config()) + + deleteObject := func(bucket, key, versionId *string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: bucket, + Key: key, + VersionId: versionId, + }) + cancel() + if err != nil { + return fmt.Errorf("failed to delete object %v: %v", *key, err) + } + return nil + } + + in := &s3.ListObjectsV2Input{Bucket: &bucket} + for { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectsV2(ctx, in) + cancel() + if err != nil { + return fmt.Errorf("failed to list objects: %v", err) + } + + for _, item := range out.Contents { + err = deleteObject(&bucket, item.Key, nil) + if err != nil { + return err + } + } + + if out.IsTruncated { + in.ContinuationToken = out.ContinuationToken + } else { + break + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + return err +} + +func TestMakeBucket(s *S3Conf) { + testname := "test make bucket" + runF(testname) + + bucket := "testbucket" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) + + testname = "test delete empty bucket" + runF(testname) + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func TestPutGetObject(s *S3Conf) { + testname := "test put/get object" + runF(testname) + + bucket := "testbucket1" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + // use funny size to prevent accidental alignments + datalen := 1234567 + data := make([]byte, datalen) + rand.Read(data) + csum := sha256.Sum256(data) + r := bytes.NewReader(data) + + name := "myobject" + s3client := s3.NewFromConfig(s.Config()) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &name, + Body: r, + }) + cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &name, + }) + defer cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + defer out.Body.Close() + + if out.ContentLength != int64(datalen) { + failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) + return + } + + b, err := io.ReadAll(out.Body) + if err != nil { + failF("%v: read body %v", testname, err) + return + } + + newsum := sha256.Sum256(b) + if csum != newsum { + failF("%v: checksum got %x expected %x", testname, newsum, csum) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func TestPutGetMPObject(s *S3Conf) { + testname := "test put/get multipart object" + runF(testname) + + bucket := "testbucket2" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + name := "mympuobject" + s3client := s3.NewFromConfig(s.Config()) + + datalen := 10*1024*1024 + 15 + dr := NewDataReader(datalen, 5*1024*1024) + WithPartSize(5 * 1024 * 1024) + s.PartSize = 5 * 1024 * 1024 + err = uploadData(s, dr, bucket, name) + if err != nil { + failF("%v: %v", testname, err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &name, + }) + defer cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + defer out.Body.Close() + + if out.ContentLength != int64(datalen) { + failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) + return + } + + b := make([]byte, 1048576) + h := sha256.New() + for { + n, err := out.Body.Read(b) + if err == io.EOF { + h.Write(b[:n]) + break + } + if err != nil { + failF("%v: read %v", err) + return + } + h.Write(b[:n]) + } + + if !isEqual(dr.Sum(), h.Sum(nil)) { + failF("%v: checksum got %x expected %x", testname, h.Sum(nil), dr.Sum()) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func isEqual(a, b []byte) bool { + if len(a) != len(b) { + return false + } + + for i, d := range a { + if d != b[i] { + return false + } + } + + return true +} + +func uploadData(s *S3Conf, r io.Reader, bucket, object string) error { + uploader := manager.NewUploader(s3.NewFromConfig(s.Config())) + uploader.PartSize = s.PartSize + uploader.Concurrency = s.Concurrency + + upinfo := &s3.PutObjectInput{ + Body: r, + Bucket: &bucket, + Key: &object, + } + + _, err := uploader.Upload(context.Background(), upinfo) + return err +} + +func TestPutDirObject(s *S3Conf) { + testname := "test put directory object" + runF(testname) + + bucket := "testbucket3" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + name := "myobjectdir/" + s3client := s3.NewFromConfig(s.Config()) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &name, + }) + cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err != nil { + failF("failed to list objects: %v", err) + return + } + + if !contains(name, out.Contents) { + failF("directory object not found") + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func TestListObject(s *S3Conf) { + testname := "list objects" + runF(testname) + + bucket := "testbucket4" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + s3client := s3.NewFromConfig(s.Config()) + + dir1 := "myobjectdir/" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &dir1, + }) + cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + + obj1 := "myobjectdir/myobject" + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj1, + }) + cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + + obj2 := "myobjectdir1/myobject" + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj2, + }) + cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + + // put: + // "myobjectdir/" + // "myobjectdir/myobject" + // "myobjectdir1/myobject" + // should return: + // "myobjectdir/myobject" + // "myobjectdir1/myobject" + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err != nil { + failF("failed to list objects: %v", err) + return + } + + if !contains(obj1, out.Contents) { + failF("object %v not found", obj1) + return + } + if !contains(obj2, out.Contents) { + failF("object %v not found", obj2) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj1, + }) + cancel() + if err != nil { + failF("failed to delete %v: %v", obj1, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj2, + }) + cancel() + if err != nil { + failF("failed to delete %v: %v", obj2, err) + return + } + + // put: + // "myobjectdir/" + // "myobjectdir/myobject" + // "myobjectdir1/myobject" + // delete: + // "myobjectdir/myobject" + // "myobjectdir1/myobject" + // should return: + // "myobjectdir/" + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + cancel() + if err != nil { + failF("failed to list objects: %v", err) + return + } + + if !contains(dir1, out.Contents) { + failF("dir %v not found", dir1) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func contains(name string, list []types.Object) bool { + for _, item := range list { + fmt.Println(*item.Key) + if strings.EqualFold(name, *item.Key) { + return true + } + } + return false +} + +func TestListAbortMultiPartObject(s *S3Conf) { + testname := "list/abort multipart objects" + runF(testname) + + bucket := "testbucket6" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + s3client := s3.NewFromConfig(s.Config()) + + obj := "mympuobject" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + failF("%v: create multipart upload: %v", testname, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + lmpu, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + failF("%v: list multipart upload: %v", testname, err) + return + } + + //for _, item := range lmpu.Uploads { + // fmt.Println(" -- ", *item.Key, *item.UploadId) + //} + + if !containsUID(obj, *mpu.UploadId, lmpu.Uploads) { + failF("%v: upload %v/%v not found", testname, obj, *mpu.UploadId) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mpu.UploadId, + }) + cancel() + if err != nil { + failF("%v: abort multipart upload: %v", testname, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + lmpu, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + failF("%v: list multipart upload: %v", testname, err) + return + } + + if len(lmpu.Uploads) != 0 { + for _, item := range lmpu.Uploads { + fmt.Println(" D- ", *item.Key, *item.UploadId) + } + failF("%v: unexpected multipart uploads found", testname) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func containsUID(name, id string, list []types.MultipartUpload) bool { + for _, item := range list { + if strings.EqualFold(name, *item.Key) && strings.EqualFold(id, *item.UploadId) { + return true + } + } + return false +} + +func TestListMultiParts(s *S3Conf) { + testname := "list multipart parts" + runF(testname) + + bucket := "testbucket7" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + s3client := s3.NewFromConfig(s.Config()) + + obj := "mympuobject" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + failF("%v: create multipart upload: %v", testname, err) + return + } + + // check list parts of no parts is good + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mpu.UploadId, + }) + cancel() + if err != nil { + failF("%v: list parts: %v", testname, err) + return + } + + if len(lp.Parts) != 0 { + failF("%v: list parts: expected no parts, got %v", + testname, len(lp.Parts)) + return + } + + // upload 1 part and check list parts + size5MB := 5 * 1024 * 1024 + dr := NewDataReader(size5MB, size5MB) + + datafile := "rand.data" + w, err := os.Create(datafile) + if err != nil { + failF("%v: create %v: %v", testname, datafile, err) + return + } + defer w.Close() + + _, err = io.Copy(w, dr) + if err != nil { + failF("%v: write %v: %v", testname, datafile, err) + return + } + + _, err = w.Seek(0, io.SeekStart) + if err != nil { + failF("%v: seek %v: %v", testname, datafile, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: 42, + UploadId: mpu.UploadId, + Body: w, + ContentLength: int64(size5MB), + }) + cancel() + if err != nil { + failF("%v: multipart put part: %v", testname, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + lp, err = s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mpu.UploadId, + }) + cancel() + if err != nil { + failF("%v: list parts: %v", testname, err) + return + } + + //for _, part := range lp.Parts { + // fmt.Println(" -- ", part.PartNumber, part.ETag) + //} + + if len(lp.Parts) != 1 || lp.Parts[0].PartNumber != 42 { + fmt.Printf("%+v, %v, %v\n", lp.Parts, *lp.Key, *lp.UploadId) + failF("%v: list parts: unexpected parts listing", testname) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func TestIncorrectMultiParts(s *S3Conf) { + testname := "incorrect multipart parts" + runF(testname) + + bucket := "testbucket8" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + s3client := s3.NewFromConfig(s.Config()) + + obj := "mympuobject" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + failF("%v: create multipart upload: %v", testname, err) + return + } + + // upload 2 parts + size5MB := 5 * 1024 * 1024 + dr := NewDataReader(size5MB, size5MB) + + datafile := "rand.data" + w, err := os.Create(datafile) + if err != nil { + failF("%v: create %v: %v", testname, datafile, err) + return + } + defer w.Close() + + _, err = io.Copy(w, dr) + if err != nil { + failF("%v: write %v: %v", testname, datafile, err) + return + } + + _, err = w.Seek(0, io.SeekStart) + if err != nil { + failF("%v: seek %v: %v", testname, datafile, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + mp1, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: 42, + UploadId: mpu.UploadId, + Body: w, + ContentLength: int64(size5MB), + }) + cancel() + if err != nil { + failF("%v: multipart put part 1: %v", testname, err) + return + } + + _, err = w.Seek(0, io.SeekStart) + if err != nil { + failF("%v: seek %v: %v", testname, datafile, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + mp2, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: 96, + UploadId: mpu.UploadId, + Body: w, + ContentLength: int64(size5MB), + }) + cancel() + if err != nil { + failF("%v: multipart put part 2: %v", testname, err) + return + } + + badEtag := "bogusEtagValue" + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mpu.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: mp2.ETag, + PartNumber: 96, + }, + { + ETag: &badEtag, + PartNumber: 99, + }, + }, + }, + }) + cancel() + if err == nil { + failF("%v: complete multipart expected err", testname) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mpu.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: mp1.ETag, + PartNumber: 42, + }, + { + ETag: mp2.ETag, + PartNumber: 96, + }, + }, + }, + }) + cancel() + if err != nil { + failF("%v: complete multipart: %v", testname, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + oi, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + failF("%v: head object %v: %v", testname, obj, err) + return + } + + if oi.ContentLength != (int64(size5MB) * 2) { + failF("%v: object len expected %v, got %v", + testname, int64(size5MB)*2, oi.ContentLength) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func TestIncompleteMultiParts(s *S3Conf) { + testname := "incomplete multipart parts" + runF(testname) + + bucket := "testbucket9" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + s3client := s3.NewFromConfig(s.Config()) + + obj := "mympuobject" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + failF("%v: create multipart upload: %v", testname, err) + return + } + + // upload 2 parts + size5MB := 5 * 1024 * 1024 + size1MB := 1024 * 1024 + dr := NewDataReader(size1MB, size1MB) + + datafile := "rand.data" + w, err := os.Create(datafile) + if err != nil { + failF("%v: create %v: %v", testname, datafile, err) + return + } + defer w.Close() + + _, err = io.Copy(w, dr) + if err != nil { + failF("%v: write %v: %v", testname, datafile, err) + return + } + + _, err = w.Seek(0, io.SeekStart) + if err != nil { + failF("%v: seek %v: %v", testname, datafile, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: 42, + UploadId: mpu.UploadId, + Body: w, + ContentLength: int64(size5MB), + }) + cancel() + if err == nil { + failF("%v: multipart put short part expected error", testname) + return + } + + // check list parts does not have incomplete part + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mpu.UploadId, + }) + cancel() + if err != nil { + failF("%v: list parts: %v", testname, err) + return + } + + if containsPart(42, lp.Parts) { + failF("%v: list parts: found incomplete part", testname) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func containsPart(part int32, list []types.Part) bool { + for _, item := range list { + if item.PartNumber == part { + return true + } + } + return false +} + +func TestIncompletePutObject(s *S3Conf) { + testname := "test incomplete put object" + runF(testname) + + bucket := "testbucket10" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + // use funny size to prevent accidental alignments + datalen := 1234567 + shortdatalen := 12345 + data := make([]byte, shortdatalen) + rand.Read(data) + r := bytes.NewReader(data) + + name := "myobject" + s3client := s3.NewFromConfig(s.Config()) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &name, + Body: r, + ContentLength: int64(datalen), + }) + cancel() + if err == nil { + failF("%v: expected error for short data put", testname) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &name, + }) + defer cancel() + if err == nil { + failF("%v: expected object not exist", testname) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func TestRangeGet(s *S3Conf) { + testname := "test range get" + runF(testname) + + bucket := "testbucket11" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + datalen := 10 * 1024 + data := make([]byte, datalen) + rand.Read(data) + r := bytes.NewReader(data) + + name := "myobject" + s3client := s3.NewFromConfig(s.Config()) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &name, + Body: r, + }) + cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + + rangeString := "bytes=100-200" + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &name, + Range: &rangeString, + }) + defer cancel() + if err != nil { + failF("%v: %v", testname, err) + return + } + defer out.Body.Close() + + b, err := io.ReadAll(out.Body) + if err != nil { + failF("%v: read body %v", testname, err) + return + } + + // bytes range is inclusive, go range for second value is not + if !isSame(b, data[100:201]) { + failF("%v: data mismatch of range", testname) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +func isSame(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i, x := range a { + if x != b[i] { + return false + } + } + return true +} + +func TestInvalidMultiParts(s *S3Conf) { + testname := "invalid multipart parts" + runF(testname) + + bucket := "bucket12" + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + + s3client := s3.NewFromConfig(s.Config()) + + obj := "mympuobject" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + failF("%v: create multipart upload: %v", testname, err) + return + } + + // upload 2 parts + size5MB := 5 * 1024 * 1024 + dr := NewDataReader(size5MB, size5MB) + + datafile := "rand.data" + w, err := os.Create(datafile) + if err != nil { + failF("%v: create %v: %v", testname, datafile, err) + return + } + defer w.Close() + + _, err = io.Copy(w, dr) + if err != nil { + failF("%v: write %v: %v", testname, datafile, err) + return + } + + _, err = w.Seek(0, io.SeekStart) + if err != nil { + failF("%v: seek %v: %v", testname, datafile, err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: -1, + UploadId: mpu.UploadId, + Body: w, + ContentLength: int64(size5MB), + }) + cancel() + if err == nil { + failF("%v: multipart put part 1 expected error", testname) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err == nil { + failF("%v: head object %v expected error", testname, obj) + return + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testname, err) + return + } + passF(testname) +} + +// Full flow test +func TestFullFlow(s *S3Conf) { + // TODO: add more test cases to get 100% coverage + TestMakeBucket(s) + TestPutGetObject(s) + TestPutGetMPObject(s) + TestPutDirObject(s) + TestListObject(s) + TestIncompletePutObject(s) + TestListMultiParts(s) + TestIncompleteMultiParts(s) + TestIncorrectMultiParts(s) + TestListAbortMultiPartObject(s) + TestListAbortMultiPartObject(s) + TestInvalidMultiParts(s) +} From 672027f4aac2e9fcbcdd9a7c1fde9a44bb2ac5fc Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Thu, 22 Jun 2023 20:03:50 +0400 Subject: [PATCH 2/2] fix: TLS configuration removed --- integration/s3conf.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/integration/s3conf.go b/integration/s3conf.go index 4bb02a8c..2533d49e 100644 --- a/integration/s3conf.go +++ b/integration/s3conf.go @@ -2,7 +2,6 @@ package integration import ( "context" - "crypto/tls" "log" "net/http" "os" @@ -92,9 +91,7 @@ func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) func (c *S3Conf) Config() aws.Config { creds := c.getCreds() - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } + tr := &http.Transport{} client := &http.Client{Transport: tr} opts := []func(*config.LoadOptions) error{