mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-24 00:52:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #194 from versity/int-tests-restruct

Browse Source 
Integration tests restructuring
This commit is contained in:
Ben McClelland
2023-08-23 09:40:07 -07:00
committed by GitHub
parent da99990225 a05179b14f
commit 56f452f1a2
4 changed files with 2722 additions and 1534 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/versitygw/test.go
-108
View File
@@ -67,114 +67,6 @@ func initTestFlags() []cli.Flag {
func initTestCommands() []*cli.Command {
return []*cli.Command{
{
Name: "bucket-actions",
Usage: "Test bucket creation, checking the existence, deletes it.",
Description: `Calls s3 gateway create-bucket action to create a new bucket,
calls head-bucket action to check the existence, then calls delete-bucket action to delete the bucket.`,
Action: getAction(integration.TestMakeBucket),
},
{
Name: "object-actions",
Usage: "Test put/get/delete/copy objects.",
Description: `Creates a bucket with s3 gateway action, puts an object in it,
tries to copy into another bucket, that doesn't exist, creates the destination bucket for copying,
copies the object, get's the object to check the length and content,
get's the copied object to check the length and content, deletes all the objects inside the source bucket,
deletes both the objects and buckets.`,
Action: getAction(integration.TestPutGetObject),
},
{
Name: "put-get-mp-object",
Usage: "Test put & get multipart object.",
Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload,
gets the object from the bucket, deletes both the object and bucket.`,
Action: getAction(integration.TestPutGetMPObject),
},
{
Name: "put-dir-object",
Usage: "Test put directory object.",
Description: `Creates a bucket with s3 gateway action, puts a directory object in it,
lists the bucket's objects, deletes both the objects and bucket.`,
Action: getAction(integration.TestPutDirObject),
},
{
Name: "list-objects",
Usage: "Test list-objects action.",
Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it,
lists the bucket's objects, deletes both the objects and bucket.`,
Action: getAction(integration.TestListObject),
},
{
Name: "abort-mp",
Usage: "Tests abort-multipart-upload action.",
Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
lists the multipart upload, aborts the multipart upload, lists the multipart upload again,
deletes both the objects and bucket.`,
Action: getAction(integration.TestListAbortMultiPartObject),
},
{
Name: "list-parts",
Usage: "Tests list-parts action.",
Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
lists the upload parts, deletes both the objects and bucket.`,
Action: getAction(integration.TestListMultiParts),
},
{
Name: "incorrect-mp",
Usage: "Tests incorrect multipart case.",
Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
uploads different parts, completes the multipart upload with incorrect part numbers,
calls the head-object action, compares the content length, removes both the object and bucket`,
Action: getAction(integration.TestIncorrectMultiParts),
},
{
Name: "incomplete-mp",
Usage: "Tests incomplete multi parts.",
Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
upload a part, lists the parts, checks if the uploaded part is in the list,
removes both the object and the bucket`,
Action: getAction(integration.TestIncompleteMultiParts),
},
{
Name: "incomplete-put-object",
Usage: "Tests incomplete put objects case.",
Description: `Creates a bucket with s3 gateway action, puts an object in it,
gets the object with head-object action, expects the object to be got,
removes both the object and bucket`,
Action: getAction(integration.TestIncompletePutObject),
},
{
Name: "get-range",
Usage: "Tests get object by range.",
Description: `Creates a bucket with s3 gateway action, puts an object in it,
gets the object by specifying the object range, compares the range with the original one,
removes both the object and the bucket`,
Action: getAction(integration.TestRangeGet),
},
{
Name: "invalid-mp",
Usage: "Tests invalid multi part case.",
Description: `Creates a bucket with s3 gateway action, creates a multi part upload,
uploads an invalid part, gets the object with head-object action, expects to get error,
removes both the object and bucket`,
Action: getAction(integration.TestInvalidMultiParts),
},
{
Name: "object-tag-actions",
Usage: "Tests get/put/delete object tag actions.",
Description: `Creates a bucket with s3 gateway action, puts an object in it,
puts some tags for the object, gets the tags, compares the results, removes the tags,
gets the tags again, checks it to be empty, then removes both the object and bucket`,
Action: getAction(integration.TestPutGetRemoveTags),
},
{
Name: "bucket-acl-actions",
Usage: "Tests put/get bucket actions.",
Description: `Creates a bucket with s3 gateway action, puts some bucket acls
gets the acl, verifies it, then removes the bucket`,
Action: getAction(integration.TestAclActions),
},
{
Name: "full-flow",
Usage: "Tests the full flow of gateway.",
integration/action-tests.go
+170
View File
@@ -0,0 +1,170 @@
package integration
func TestCreateBucket(s *S3Conf) {
CreateBucket_invalid_bucket_name(s)
CreateBucket_existing_bucket(s)
CreateDeleteBucket_success(s)
}
func TestHeadBucket(s *S3Conf) {
HeadBucket_non_existing_bucket(s)
HeadBucket_success(s)
}
func TestDeleteBucket(s *S3Conf) {
DeleteBucket_non_existing_bucket(s)
DeleteBucket_non_empty_bucket(s)
}
func TestPutObject(s *S3Conf) {
PutObject_non_existing_bucket(s)
PutObject_special_chars(s)
PutObject_existing_dir_obj(s)
PutObject_obj_parent_is_file(s)
PutObject_success(s)
}
func TestHeadObject(s *S3Conf) {
HeadObject_non_existing_object(s)
HeadObject_success(s)
}
func TestGetObject(s *S3Conf) {
GetObject_non_existing_key(s)
GetObject_invalid_ranges(s)
GetObject_with_meta(s)
GetObject_success(s)
GetObject_by_range_success(s)
}
func TestListObjects(s *S3Conf) {
ListObjects_non_existing_bucket(s)
ListObjects_with_prefix(s)
ListObject_truncated(s)
ListObjects_invalid_max_keys(s)
ListObjects_max_keys_0(s)
}
func TestDeleteObject(s *S3Conf) {
DeleteObject_non_existing_object(s)
DeleteObject_success(s)
}
func TestDeleteObjects(s *S3Conf) {
DeleteObjects_empty_input(s)
//TODO: Uncomment this after fixing the bug: #195
// DeleteObjects_non_existing_objects(s)
DeleteObjects_success(s)
}
func TestCopyObject(s *S3Conf) {
CopyObject_non_existing_dst_bucket(s)
CopyObject_success(s)
}
func TestPutObjectTagging(s *S3Conf) {
PutObjectTagging_non_existing_object(s)
PutObjectTagging_success(s)
}
func TestGetObjectTagging(s *S3Conf) {
GetObjectTagging_non_existing_object(s)
GetObjectTagging_success(s)
}
func TestDeleteObjectTagging(s *S3Conf) {
DeleteObjectTagging_non_existing_object(s)
DeleteObjectTagging_success(s)
}
func TestCreateMultipartUpload(s *S3Conf) {
CreateMultipartUpload_non_existing_bucket(s)
CreateMultipartUpload_success(s)
}
func TestUploadPart(s *S3Conf) {
UploadPart_non_existing_bucket(s)
UploadPart_invalid_part_number(s)
UploadPart_non_existing_key(s)
UploadPart_non_existing_mp_upload(s)
UploadPart_success(s)
}
func TestUploadPartCopy(s *S3Conf) {
UploadPartCopy_non_existing_bucket(s)
UploadPartCopy_incorrect_uploadId(s)
UploadPartCopy_incorrect_object_key(s)
UploadPartCopy_invalid_part_number(s)
UploadPartCopy_invalid_copy_source(s)
UploadPartCopy_non_existing_source_bucket(s)
UploadPartCopy_non_existing_source_object_key(s)
UploadPartCopy_success(s)
UploadPartCopy_by_range_invalid_range(s)
UploadPartCopy_by_range_success(s)
}
func TestListParts(s *S3Conf) {
ListParts_incorrect_uploadId(s)
ListParts_incorrect_object_key(s)
ListParts_success(s)
}
func TestListMultipartUploads(s *S3Conf) {
ListMultipartUploads_non_existing_bucket(s)
ListMultipartUploads_empty_result(s)
ListMultipartUploads_success(s)
}
func TestAbortMultipartUpload(s *S3Conf) {
AbortMultipartUpload_non_existing_bucket(s)
AbortMultipartUpload_incorrect_uploadId(s)
AbortMultipartUpload_incorrect_object_key(s)
AbortMultipartUpload_success(s)
}
func TestCompleteMultipartUpload(s *S3Conf) {
CompletedMultipartUpload_non_existing_bucket(s)
CompleteMultipartUpload_invalid_part_number(s)
CompleteMultipartUpload_invalid_ETag(s)
CompleteMultipartUpload_success(s)
}
func TestPutBucketAcl(s *S3Conf) {
PutBucketAcl_non_existing_bucket(s)
PutBucketAcl_invalid_acl_canned_and_acp(s)
PutBucketAcl_invalid_acl_canned_and_grants(s)
PutBucketAcl_invalid_acl_acp_and_grants(s)
PutBucketAcl_invalid_owner(s)
PutBucketAcl_success_access_denied(s)
PutBucketAcl_success(s)
}
func TestGetBucketAcl(s *S3Conf) {
GetBucketAcl_non_existing_bucket(s)
GetBucketAcl_access_denied(s)
GetBucketAcl_success(s)
}
func TestFullFlow(s *S3Conf) {
TestCreateBucket(s)
TestHeadBucket(s)
TestDeleteBucket(s)
TestPutObject(s)
TestHeadObject(s)
TestGetObject(s)
TestListObjects(s)
TestDeleteObject(s)
TestDeleteObjects(s)
TestCopyObject(s)
TestPutObjectTagging(s)
TestDeleteObjectTagging(s)
TestCreateMultipartUpload(s)
TestUploadPart(s)
TestUploadPartCopy(s)
TestListParts(s)
TestListMultipartUploads(s)
TestAbortMultipartUpload(s)
TestCompleteMultipartUpload(s)
TestPutBucketAcl(s)
TestGetBucketAcl(s)
}
integration/tests.go
+2314 -1407
View File
File diff suppressed because it is too large Load Diff
integration/utils.go
+238 -19
View File
@@ -1,15 +1,34 @@
package integration
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"errors"
"fmt"
"io"
"os"
"os/exec"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/versity/versitygw/s3err"
)
var (
bcktCount = 0
succUsrCrt = "The user has been created successfully"
failUsrCrt = "failed to create a user: update iam data: account already exists"
)
func getBucketName() string {
bcktCount++
return fmt.Sprintf("test-bucket-%v", bcktCount)
}
func setup(s *S3Conf, bucket string) error {
s3client := s3.NewFromConfig(s.Config())
@@ -69,6 +88,89 @@ func teardown(s *S3Conf, bucket string) error {
return err
}
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
runF(testName)
bucketName := getBucketName()
err := setup(s, bucketName)
if err != nil {
failF("%v: failed to create a bucket: %v", testName, err.Error())
return
}
client := s3.NewFromConfig(s.Config())
handlerErr := handler(client, bucketName)
if handlerErr != nil {
failF("%v: %v", testName, handlerErr.Error())
}
err = teardown(s, bucketName)
if err != nil {
if handlerErr == nil {
failF("%v: failed to delete the bucket: %v", testName, err.Error())
} else {
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
}
}
if handlerErr == nil {
passF(testName)
}
}
func putObjects(client *s3.Client, objs []string, bucket string) error {
for _, key := range objs {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := client.PutObject(ctx, &s3.PutObjectInput{
Key: &key,
Bucket: &bucket,
})
cancel()
if err != nil {
return err
}
}
return nil
}
func checkApiErr(err error, apiErr s3err.APIError) error {
if err == nil {
return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
}
var ae smithy.APIError
if errors.As(err, &ae) {
if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
return nil
}
return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
} else {
return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
}
}
func checkSdkApiErr(err error, code string) error {
var ae smithy.APIError
if errors.As(err, &ae) {
if ae.ErrorCode() != code {
return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code)
}
return nil
}
return err
}
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
data = make([]byte, lgth)
rand.Read(data)
csum = sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = client.PutObject(ctx, input)
cancel()
return
}
func isEqual(a, b []byte) bool {
if len(a) != len(b) {
return false
@@ -83,32 +185,33 @@ func isEqual(a, b []byte) bool {
return true
}
func contains(name string, list []types.Object) bool {
for _, item := range list {
fmt.Println(*item.Key)
if strings.EqualFold(name, *item.Key) {
return true
func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
if len(list1) != len(list2) {
return false
}
for i, item := range list1 {
if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
return false
}
}
return false
return true
}
func containsUID(name, id string, list []types.MultipartUpload) bool {
for _, item := range list {
if strings.EqualFold(name, *item.Key) && strings.EqualFold(id, *item.UploadId) {
return true
}
func compareParts(parts1, parts2 []types.Part) bool {
if len(parts1) != len(parts2) {
return false
}
return false
}
func containsPart(part int32, list []types.Part) bool {
for _, item := range list {
if item.PartNumber == part {
return true
for i, prt := range parts1 {
if prt.PartNumber != parts2[i].PartNumber {
return false
}
if *prt.ETag != *parts2[i].ETag {
return false
}
}
return false
return true
}
func areTagsSame(tags1, tags2 []types.Tag) bool {
@@ -133,7 +236,7 @@ func containsTag(tag types.Tag, list []types.Tag) bool {
return false
}
func checkGrants(grts1, grts2 []types.Grant) bool {
func compareGrants(grts1, grts2 []types.Grant) bool {
if len(grts1) != len(grts2) {
return false
}
@@ -162,6 +265,10 @@ func getString(str *string) string {
return *str
}
func getPtr(str string) *string {
return &str
}
func areMapsSame(mp1, mp2 map[string]string) bool {
if len(mp1) != len(mp2) {
return false
@@ -173,3 +280,115 @@ func areMapsSame(mp1, mp2 map[string]string) bool {
}
return true
}
func compareObjects(list1 []string, list2 []types.Object) bool {
if len(list1) != len(list2) {
return false
}
elementMap := make(map[string]bool)
for _, elem := range list1 {
elementMap[elem] = true
}
for _, elem := range list2 {
if _, found := elementMap[*elem.Key]; !found {
return false
}
}
return true
}
func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
if len(list1) != len(list2) {
return false
}
elementMap := make(map[string]bool)
for _, elem := range list1 {
elementMap[elem] = true
}
for _, elem := range list2 {
if _, found := elementMap[*elem.Key]; !found {
return false
}
}
return true
}
func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
dr := NewDataReader(size, size)
datafile := "rand.data"
w, err := os.Create(datafile)
if err != nil {
return parts, err
}
defer w.Close()
_, err = io.Copy(w, dr)
if err != nil {
return parts, err
}
fileInfo, err := w.Stat()
if err != nil {
return parts, err
}
partSize := fileInfo.Size() / int64(partCount)
var offset int64
for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
partStart := (partNumber - 1) * partSize
partEnd := partStart + partSize - 1
if partEnd > fileInfo.Size()-1 {
partEnd = fileInfo.Size() - 1
}
partBuffer := make([]byte, partEnd-partStart+1)
_, err := w.ReadAt(partBuffer, partStart)
if err != nil {
return parts, err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: int32(partNumber),
})
cancel()
if err != nil {
return parts, err
} else {
parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
offset += partSize
}
}
return parts, err
}
type user struct {
access string
secret string
role string
}
func createUsers(s *S3Conf, users []user) error {
for _, usr := range users {
out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
if err != nil {
return err
}
if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
return fmt.Errorf("failed to create a user account")
}
}
return nil
}