mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-05 23:57:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: get simple multipart upload tests passing

Browse Source
This commit is contained in:
Ben McClelland
2023-06-12 07:37:21 -07:00
parent 2697edd40a
commit ef06d11d7c
11 changed files with 257 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3api/controllers/backend_moq_test.go
+7 -6
View File
@@ -7,6 +7,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3response"
"io"
"sync"
)
@@ -69,10 +70,10 @@ var _ backend.Backend = &BackendMock{}
// ListBucketsFunc: func() (*s3.ListBucketsOutput, error) {
// panic("mock out the ListBuckets method")
// },
// ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) {
// ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
// panic("mock out the ListMultipartUploads method")
// },
// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
// panic("mock out the ListObjectParts method")
// },
// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
@@ -173,10 +174,10 @@ type BackendMock struct {
ListBucketsFunc func() (*s3.ListBucketsOutput, error)
// ListMultipartUploadsFunc mocks the ListMultipartUploads method.
ListMultipartUploadsFunc func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error)
ListMultipartUploadsFunc func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
// ListObjectPartsFunc mocks the ListObjectParts method.
ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error)
ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error)
// ListObjectsFunc mocks the ListObjects method.
ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error)
@@ -1095,7 +1096,7 @@ func (mock *BackendMock) ListBucketsCalls() []struct {
}
// ListMultipartUploads calls ListMultipartUploadsFunc.
func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) {
func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
if mock.ListMultipartUploadsFunc == nil {
panic("BackendMock.ListMultipartUploadsFunc: method is nil but Backend.ListMultipartUploads was just called")
}
@@ -1127,7 +1128,7 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct {
}
// ListObjectParts calls ListObjectPartsFunc.
func (mock *BackendMock) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
func (mock *BackendMock) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
if mock.ListObjectPartsFunc == nil {
panic("BackendMock.ListObjectPartsFunc: method is nil but Backend.ListObjectParts was just called")
}
s3api/controllers/base.go
+32 -50
View File
@@ -24,7 +24,6 @@ import (
"os"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -53,25 +52,21 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
key := ctx.Params("key")
keyEnd := ctx.Params("*1")
uploadId := ctx.Query("uploadId")
maxPartsStr := ctx.Query("max-parts")
partNumberMarkerStr := ctx.Query("part-number-marker")
maxParts := ctx.QueryInt("max-parts", 0)
partNumberMarker := ctx.QueryInt("part-number-marker", 0)
acceptRange := ctx.Get("Range")
if keyEnd != "" {
key = strings.Join([]string{key, keyEnd}, "/")
}
if uploadId != "" {
maxParts, err := strconv.Atoi(maxPartsStr)
if err != nil && maxPartsStr != "" {
return errors.New("wrong api call")
if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") {
return ErrorResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts))
}
partNumberMarker, err := strconv.Atoi(partNumberMarkerStr)
if err != nil && partNumberMarkerStr != "" {
return errors.New("wrong api call")
if partNumberMarker < 0 || (partNumberMarker == 0 && ctx.Query("part-number-marker") != "") {
return ErrorResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker))
}
res, err := c.be.ListObjectParts(bucket, "", uploadId, partNumberMarker, maxParts)
res, err := c.be.ListObjectParts(bucket, key, uploadId, partNumberMarker, maxParts)
return Responce(ctx, res, err)
}
@@ -186,40 +181,24 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
keyStart = keyStart + "/"
}
if partNumberStr != "" {
copySrcModifSinceDate, err := time.Parse(time.RFC3339, copySrcModifSince)
if err != nil && copySrcModifSince != "" {
return errors.New("wrong api call")
}
copySrcUnmodifSinceDate, err := time.Parse(time.RFC3339, copySrcUnmodifSince)
if err != nil && copySrcUnmodifSince != "" {
return errors.New("wrong api call")
}
partNumber, err := strconv.ParseInt(partNumberStr, 10, 64)
var contentLength int64
if contentLengthStr != "" {
var err error
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return errors.New("wrong api call")
return ErrorResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest))
}
res, err := c.be.UploadPartCopy(&s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &keyStart,
PartNumber: int32(partNumber),
UploadId: &uploadId,
CopySource: &copySource,
CopySourceIfMatch: &copySrcIfMatch,
CopySourceIfNoneMatch: &copySrcIfNoneMatch,
CopySourceIfModifiedSince: &copySrcModifSinceDate,
CopySourceIfUnmodifiedSince: &copySrcUnmodifSinceDate,
})
return Responce(ctx, res, err)
}
if uploadId != "" {
if uploadId != "" && partNumberStr != "" {
partNumber := ctx.QueryInt("partNumber", -1)
if partNumber < 1 {
return ErrorResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart))
}
body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
res, err := c.be.UploadPart(bucket, keyStart, uploadId, body)
res, err := c.be.PutObjectPart(bucket, keyStart, uploadId,
partNumber, contentLength, body)
return Responce(ctx, res, err)
}
@@ -242,6 +221,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
if copySource != "" {
_, _, _, _ = copySrcIfMatch, copySrcIfNoneMatch,
copySrcModifSince, copySrcUnmodifSince
copySourceSplit := strings.Split(copySource, "/")
srcBucket, srcObject := copySourceSplit[0], copySourceSplit[1:]
@@ -249,11 +230,6 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return Responce(ctx, res, err)
}
contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return errors.New("wrong api call")
}
metadata := utils.GetUserMetaData(&ctx.Request().Header)
res, err := c.be.PutObject(&s3.PutObjectInput{
@@ -381,20 +357,22 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
}
if uploadId != "" {
var parts []types.Part
data := struct {
Parts []types.Part `xml:"Part"`
}{}
if err := xml.Unmarshal(ctx.Body(), &parts); err != nil {
if err := xml.Unmarshal(ctx.Body(), &data); err != nil {
return errors.New("wrong api call")
}
res, err := c.be.CompleteMultipartUpload(bucket, "", uploadId, parts)
res, err := c.be.CompleteMultipartUpload(bucket, key, uploadId, data.Parts)
return Responce(ctx, res, err)
}
res, err := c.be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
return Responce(ctx, res, err)
}
func Responce[R comparable](ctx *fiber.Ctx, resp R, err error) error {
func Responce[R any](ctx *fiber.Ctx, resp R, err error) error {
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
@@ -414,6 +392,10 @@ func Responce[R comparable](ctx *fiber.Ctx, resp R, err error) error {
return err
}
if len(b) > 0 {
ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
}
return ctx.Send(b)
}
s3api/controllers/base_test.go
+15 -12
View File
@@ -29,6 +29,7 @@ import (
"github.com/valyala/fasthttp"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
func TestNew(t *testing.T) {
@@ -128,8 +129,8 @@ func TestS3ApiController_GetActions(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{be: &BackendMock{
ListObjectPartsFunc: func(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
return &s3.ListPartsOutput{}, nil
ListObjectPartsFunc: func(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
return s3response.ListPartsResponse{}, nil
},
GetObjectAclFunc: func(bucket, object string) (*s3.GetObjectAclOutput, error) {
return &s3.GetObjectAclOutput{}, nil
@@ -169,16 +170,16 @@ func TestS3ApiController_GetActions(t *testing.T) {
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=InvalidMaxParts", nil),
},
wantErr: false,
statusCode: 500,
statusCode: 400,
},
{
name: "Get-actions-invalid-part-number",
name: "Get-actions-invalid-part-number-marker",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=200&part-number-marker=InvalidPartNumber", nil),
},
wantErr: false,
statusCode: 500,
statusCode: 400,
},
{
name: "Get-actions-list-object-parts-success",
@@ -233,8 +234,8 @@ func TestS3ApiController_ListActions(t *testing.T) {
GetBucketAclFunc: func(bucket string) (*s3.GetBucketAclOutput, error) {
return &s3.GetBucketAclOutput{}, nil
},
ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) {
return &s3.ListMultipartUploadsOutput{}, nil
ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
return s3response.ListMultipartUploadsResponse{}, nil
},
ListObjectsV2Func: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
return &s3.ListObjectsV2Output{}, nil
@@ -441,13 +442,13 @@ func TestS3ApiController_PutActions(t *testing.T) {
statusCode int
}{
{
name: "Upload-copy-part-error-case",
name: "Upload-put-part-error-case",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?partNumber=invalid", nil),
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=abc&partNumber=invalid", nil),
},
wantErr: false,
statusCode: 500,
statusCode: 400,
},
{
name: "Upload-copy-part-success",
@@ -517,11 +518,13 @@ func TestS3ApiController_PutActions(t *testing.T) {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("S3ApiController.GetActions() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("S3ApiController.GetActions() %v error = %v, wantErr %v",
tt.name, err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("S3ApiController.GetActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
t.Errorf("S3ApiController.GetActions() %v statusCode = %v, wantStatusCode = %v",
tt.name, resp.StatusCode, tt.statusCode)
}
}
}