From ef06d11d7c755478fcf9b22c6ea1eec8a653ada6 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Tue, 6 Jun 2023 13:36:21 -0700 Subject: [PATCH] fix: get simple multipart upload tests passing --- backend/backend.go | 13 +-- backend/backend_moq_test.go | 13 +-- backend/common.go | 5 + backend/posix/posix.go | 139 ++++++++++++++++---------- backend/posix/posix_darwin.go | 2 +- backend/posix/posix_linux.go | 2 +- cmd/versitygw/main.go | 1 + s3api/controllers/backend_moq_test.go | 13 +-- s3api/controllers/base.go | 82 ++++++--------- s3api/controllers/base_test.go | 27 ++--- s3response/s3response.go | 96 ++++++++++++++++++ 11 files changed, 257 insertions(+), 136 deletions(-) create mode 100644 s3response/s3response.go diff --git a/backend/backend.go b/backend/backend.go index caa6bac..12e31d4 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -21,6 +21,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" ) //go:generate moq -out backend_moq_test.go . Backend @@ -39,8 +40,8 @@ type Backend interface { CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) AbortMultipartUpload(*s3.AbortMultipartUploadInput) error - ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) - ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) + ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) + ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) @@ -115,11 +116,11 @@ func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID strin func (BackendUnsupported) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { + return s3response.ListMultipartUploadsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { + return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) diff --git a/backend/backend_moq_test.go b/backend/backend_moq_test.go index d3f2e4d..6ecfc5b 100644 --- a/backend/backend_moq_test.go +++ b/backend/backend_moq_test.go @@ -6,6 +6,7 @@ package backend import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3response" "io" "sync" ) @@ -68,10 +69,10 @@ var _ Backend = &BackendMock{} // ListBucketsFunc: func() (*s3.ListBucketsOutput, error) { // panic("mock out the ListBuckets method") // }, -// ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) { +// ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { // panic("mock out the ListMultipartUploads method") // }, -// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { +// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { // panic("mock out the ListObjectParts method") // }, // ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { @@ -172,10 +173,10 @@ type BackendMock struct { ListBucketsFunc func() (*s3.ListBucketsOutput, error) // ListMultipartUploadsFunc mocks the ListMultipartUploads method. - ListMultipartUploadsFunc func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) + ListMultipartUploadsFunc func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) // ListObjectPartsFunc mocks the ListObjectParts method. - ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) + ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) // ListObjectsFunc mocks the ListObjects method. ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) @@ -1094,7 +1095,7 @@ func (mock *BackendMock) ListBucketsCalls() []struct { } // ListMultipartUploads calls ListMultipartUploadsFunc. -func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) { +func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { if mock.ListMultipartUploadsFunc == nil { panic("BackendMock.ListMultipartUploadsFunc: method is nil but Backend.ListMultipartUploads was just called") } @@ -1126,7 +1127,7 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct { } // ListObjectParts calls ListObjectPartsFunc. -func (mock *BackendMock) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { +func (mock *BackendMock) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { if mock.ListObjectPartsFunc == nil { panic("BackendMock.ListObjectPartsFunc: method is nil but Backend.ListObjectParts was just called") } diff --git a/backend/common.go b/backend/common.go index af18ad4..6777207 100644 --- a/backend/common.go +++ b/backend/common.go @@ -24,6 +24,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" ) +var ( + // RFC3339TimeFormat RFC3339 time format + RFC3339TimeFormat = "2006-01-02T15:04:05.999Z" +) + func IsValidBucketName(name string) bool { return true } type ByBucketName []types.Bucket diff --git a/backend/posix/posix.go b/backend/posix/posix.go index a5994ec..68cd269 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -36,6 +36,7 @@ import ( "github.com/pkg/xattr" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" ) type Posix struct { @@ -231,8 +232,10 @@ func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts [ // check all parts ok last := len(parts) - 1 partsize := int64(0) + var totalsize int64 for i, p := range parts { - fi, err := os.Lstat(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))) + partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)) + fi, err := os.Lstat(partPath) if err != nil { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } @@ -240,13 +243,21 @@ func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts [ if i == 0 { partsize = fi.Size() } + totalsize += fi.Size() // all parts except the last need to be the same size if i < last && partsize != fi.Size() { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } + + b, err := xattr.Get(partPath, "user.etag") + etag := string(b) + if err != nil { + etag = "" + } + parts[i].ETag = &etag } - f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0) + f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, totalsize) if err != nil { return nil, fmt.Errorf("open temp file: %w", err) } @@ -272,11 +283,8 @@ func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts [ dir := filepath.Dir(objname) if dir != "" { if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil { - if err != nil && os.IsExist(err) { - return nil, s3err.GetAPIError(s3err.ErrObjectParentIsFile) - } if err != nil { - return nil, fmt.Errorf("make object parent directories: %w", err) + return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) } } } @@ -479,24 +487,40 @@ func (p *Posix) AbortMultipartUpload(mpu *s3.AbortMultipartUploadInput) error { return nil } -func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) { +func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { bucket := *mpu.Bucket + var delimiter string + if mpu.Delimiter != nil { + delimiter = *mpu.Delimiter + } + var prefix string + if mpu.Prefix != nil { + prefix = *mpu.Prefix + } + + var lmu s3response.ListMultipartUploadsResponse _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return lmu, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return nil, fmt.Errorf("stat bucket: %w", err) + return lmu, fmt.Errorf("stat bucket: %w", err) } // ignore readdir error and use the empty list returned objs, _ := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir)) - var uploads []types.MultipartUpload + var uploads []s3response.Upload - keyMarker := *mpu.KeyMarker - uploadIDMarker := *mpu.UploadIdMarker + var keyMarker string + if mpu.KeyMarker != nil { + keyMarker = *mpu.KeyMarker + } + var uploadIDMarker string + if mpu.UploadIdMarker != nil { + uploadIDMarker = *mpu.UploadIdMarker + } var pastMarker bool if keyMarker == "" && uploadIDMarker == "" { pastMarker = true @@ -512,7 +536,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.Lis continue } objectName := string(b) - if !strings.HasPrefix(objectName, *mpu.Prefix) { + if mpu.Prefix != nil && !strings.HasPrefix(objectName, *mpu.Prefix) { continue } @@ -538,64 +562,71 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.Lis upiddir := filepath.Join(bucket, metaTmpMultipartDir, obj.Name(), upid.Name()) loadUserMetaData(upiddir, userMetaData) + fi, err := upid.Info() + if err != nil { + return lmu, fmt.Errorf("stat %q: %w", upid.Name(), err) + } + uploadID := upid.Name() - uploads = append(uploads, types.MultipartUpload{ - Key: &objectName, - UploadId: &uploadID, + uploads = append(uploads, s3response.Upload{ + Key: objectName, + UploadID: uploadID, + Initiated: fi.ModTime().Format(backend.RFC3339TimeFormat), }) if len(uploads) == int(mpu.MaxUploads) { - return &s3.ListMultipartUploadsOutput{ - Bucket: &bucket, - Delimiter: mpu.Delimiter, + return s3response.ListMultipartUploadsResponse{ + Bucket: bucket, + Delimiter: delimiter, IsTruncated: i != len(objs) || j != len(upids), - KeyMarker: &keyMarker, - MaxUploads: mpu.MaxUploads, - NextKeyMarker: &objectName, - NextUploadIdMarker: &uploadID, - Prefix: mpu.Prefix, - UploadIdMarker: mpu.UploadIdMarker, + KeyMarker: keyMarker, + MaxUploads: int(mpu.MaxUploads), + NextKeyMarker: objectName, + NextUploadIDMarker: uploadID, + Prefix: prefix, + UploadIDMarker: uploadIDMarker, Uploads: uploads, }, nil } } } - return &s3.ListMultipartUploadsOutput{ - Bucket: &bucket, - Delimiter: mpu.Delimiter, - KeyMarker: &keyMarker, - MaxUploads: mpu.MaxUploads, - Prefix: mpu.Prefix, - UploadIdMarker: mpu.UploadIdMarker, + return s3response.ListMultipartUploadsResponse{ + Bucket: bucket, + Delimiter: delimiter, + KeyMarker: keyMarker, + MaxUploads: int(mpu.MaxUploads), + Prefix: prefix, + UploadIDMarker: uploadIDMarker, Uploads: uploads, }, nil } -func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { +func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { + var lpr s3response.ListPartsResponse _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return lpr, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return nil, fmt.Errorf("stat bucket: %w", err) + return lpr, fmt.Errorf("stat bucket: %w", err) } sum, err := p.checkUploadIDExists(bucket, object, uploadID) if err != nil { - return nil, err + return lpr, err } objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) ents, err := os.ReadDir(filepath.Join(objdir, uploadID)) if errors.Is(err, fs.ErrNotExist) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + return lpr, s3err.GetAPIError(s3err.ErrNoSuchUpload) } if err != nil { - return nil, fmt.Errorf("readdir upload: %w", err) + return lpr, fmt.Errorf("readdir upload: %w", err) } - var parts []types.Part + var parts []s3response.Part for _, e := range ents { pn, _ := strconv.Atoi(e.Name()) if pn <= partNumberMarker { @@ -614,10 +645,10 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke continue } - parts = append(parts, types.Part{ - PartNumber: int32(pn), - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), + parts = append(parts, s3response.Part{ + PartNumber: pn, + ETag: etag, + LastModified: fi.ModTime().Format(backend.RFC3339TimeFormat), Size: fi.Size(), }) } @@ -626,12 +657,12 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke func(i int, j int) bool { return parts[i].PartNumber < parts[j].PartNumber }) oldLen := len(parts) - if len(parts) > maxParts { + if maxParts > 0 && len(parts) > maxParts { parts = parts[:maxParts] } newLen := len(parts) - nextpart := int32(0) + nextpart := 0 if len(parts) != 0 { nextpart = parts[len(parts)-1].PartNumber } @@ -640,15 +671,15 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke upiddir := filepath.Join(objdir, uploadID) loadUserMetaData(upiddir, userMetaData) - return &s3.ListPartsOutput{ - Bucket: &bucket, + return s3response.ListPartsResponse{ + Bucket: bucket, IsTruncated: oldLen != newLen, - Key: &object, - MaxParts: int32(maxParts), - NextPartNumberMarker: backend.GetStringPtr(fmt.Sprintf("%v", nextpart)), - PartNumberMarker: backend.GetStringPtr(fmt.Sprintf("%v", partNumberMarker)), + Key: object, + MaxParts: maxParts, + NextPartNumberMarker: nextpart, + PartNumberMarker: partNumberMarker, Parts: parts, - UploadId: &uploadID, + UploadID: uploadID, }, nil } @@ -689,7 +720,7 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length } dataSum := hash.Sum(nil) - etag := hex.EncodeToString(dataSum[:]) + etag := hex.EncodeToString(dataSum) xattr.Set(partPath, "user.etag", []byte(etag)) return etag, nil @@ -741,7 +772,7 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { if dir != "" { err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key) if err != nil { - return "", fmt.Errorf("make object parent directories: %w", err) + return "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) } } diff --git a/backend/posix/posix_darwin.go b/backend/posix/posix_darwin.go index e67f93b..819d78f 100644 --- a/backend/posix/posix_darwin.go +++ b/backend/posix/posix_darwin.go @@ -76,7 +76,7 @@ func (tmp *tmpfile) link() error { func (tmp *tmpfile) Write(b []byte) (int, error) { if int64(len(b)) > tmp.size { - return 0, fmt.Errorf("write exceeds content length") + return 0, fmt.Errorf("write exceeds content length %v", tmp.size) } n, err := tmp.f.Write(b) diff --git a/backend/posix/posix_linux.go b/backend/posix/posix_linux.go index 9ce49d3..abc41ce 100644 --- a/backend/posix/posix_linux.go +++ b/backend/posix/posix_linux.go @@ -150,7 +150,7 @@ func (tmp *tmpfile) fallbackLink() error { func (tmp *tmpfile) Write(b []byte) (int, error) { if int64(len(b)) > tmp.size { - return 0, fmt.Errorf("write exceeds content length") + return 0, fmt.Errorf("write exceeds content length %v", tmp.size) } n, err := tmp.f.Write(b) diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 95e2814..ba6a286 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -136,6 +136,7 @@ func runGateway(be backend.Backend) error { app := fiber.New(fiber.Config{ AppName: "versitygw", ServerHeader: "VERSITYGW", + BodyLimit: 5 * 1024 * 1024 * 1024, }) var opts []s3api.Option diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 44df779..4e6b425 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3response" "io" "sync" ) @@ -69,10 +70,10 @@ var _ backend.Backend = &BackendMock{} // ListBucketsFunc: func() (*s3.ListBucketsOutput, error) { // panic("mock out the ListBuckets method") // }, -// ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) { +// ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { // panic("mock out the ListMultipartUploads method") // }, -// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { +// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { // panic("mock out the ListObjectParts method") // }, // ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { @@ -173,10 +174,10 @@ type BackendMock struct { ListBucketsFunc func() (*s3.ListBucketsOutput, error) // ListMultipartUploadsFunc mocks the ListMultipartUploads method. - ListMultipartUploadsFunc func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) + ListMultipartUploadsFunc func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) // ListObjectPartsFunc mocks the ListObjectParts method. - ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) + ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) // ListObjectsFunc mocks the ListObjects method. ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) @@ -1095,7 +1096,7 @@ func (mock *BackendMock) ListBucketsCalls() []struct { } // ListMultipartUploads calls ListMultipartUploadsFunc. -func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) { +func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { if mock.ListMultipartUploadsFunc == nil { panic("BackendMock.ListMultipartUploadsFunc: method is nil but Backend.ListMultipartUploads was just called") } @@ -1127,7 +1128,7 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct { } // ListObjectParts calls ListObjectPartsFunc. -func (mock *BackendMock) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { +func (mock *BackendMock) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { if mock.ListObjectPartsFunc == nil { panic("BackendMock.ListObjectPartsFunc: method is nil but Backend.ListObjectParts was just called") } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index bea3b76..da019ee 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -24,7 +24,6 @@ import ( "os" "strconv" "strings" - "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -53,25 +52,21 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { key := ctx.Params("key") keyEnd := ctx.Params("*1") uploadId := ctx.Query("uploadId") - maxPartsStr := ctx.Query("max-parts") - partNumberMarkerStr := ctx.Query("part-number-marker") + maxParts := ctx.QueryInt("max-parts", 0) + partNumberMarker := ctx.QueryInt("part-number-marker", 0) acceptRange := ctx.Get("Range") if keyEnd != "" { key = strings.Join([]string{key, keyEnd}, "/") } if uploadId != "" { - maxParts, err := strconv.Atoi(maxPartsStr) - if err != nil && maxPartsStr != "" { - return errors.New("wrong api call") + if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") { + return ErrorResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts)) } - - partNumberMarker, err := strconv.Atoi(partNumberMarkerStr) - if err != nil && partNumberMarkerStr != "" { - return errors.New("wrong api call") + if partNumberMarker < 0 || (partNumberMarker == 0 && ctx.Query("part-number-marker") != "") { + return ErrorResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker)) } - - res, err := c.be.ListObjectParts(bucket, "", uploadId, partNumberMarker, maxParts) + res, err := c.be.ListObjectParts(bucket, key, uploadId, partNumberMarker, maxParts) return Responce(ctx, res, err) } @@ -186,40 +181,24 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { keyStart = keyStart + "/" } - if partNumberStr != "" { - copySrcModifSinceDate, err := time.Parse(time.RFC3339, copySrcModifSince) - if err != nil && copySrcModifSince != "" { - return errors.New("wrong api call") - } - - copySrcUnmodifSinceDate, err := time.Parse(time.RFC3339, copySrcUnmodifSince) - if err != nil && copySrcUnmodifSince != "" { - return errors.New("wrong api call") - } - - partNumber, err := strconv.ParseInt(partNumberStr, 10, 64) + var contentLength int64 + if contentLengthStr != "" { + var err error + contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64) if err != nil { - return errors.New("wrong api call") + return ErrorResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest)) } - - res, err := c.be.UploadPartCopy(&s3.UploadPartCopyInput{ - Bucket: &bucket, - Key: &keyStart, - PartNumber: int32(partNumber), - UploadId: &uploadId, - CopySource: ©Source, - CopySourceIfMatch: ©SrcIfMatch, - CopySourceIfNoneMatch: ©SrcIfNoneMatch, - CopySourceIfModifiedSince: ©SrcModifSinceDate, - CopySourceIfUnmodifiedSince: ©SrcUnmodifSinceDate, - }) - - return Responce(ctx, res, err) } - if uploadId != "" { + if uploadId != "" && partNumberStr != "" { + partNumber := ctx.QueryInt("partNumber", -1) + if partNumber < 1 { + return ErrorResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart)) + } + body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body()))) - res, err := c.be.UploadPart(bucket, keyStart, uploadId, body) + res, err := c.be.PutObjectPart(bucket, keyStart, uploadId, + partNumber, contentLength, body) return Responce(ctx, res, err) } @@ -242,6 +221,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } if copySource != "" { + _, _, _, _ = copySrcIfMatch, copySrcIfNoneMatch, + copySrcModifSince, copySrcUnmodifSince copySourceSplit := strings.Split(copySource, "/") srcBucket, srcObject := copySourceSplit[0], copySourceSplit[1:] @@ -249,11 +230,6 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { return Responce(ctx, res, err) } - contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - return errors.New("wrong api call") - } - metadata := utils.GetUserMetaData(&ctx.Request().Header) res, err := c.be.PutObject(&s3.PutObjectInput{ @@ -381,20 +357,22 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { } if uploadId != "" { - var parts []types.Part + data := struct { + Parts []types.Part `xml:"Part"` + }{} - if err := xml.Unmarshal(ctx.Body(), &parts); err != nil { + if err := xml.Unmarshal(ctx.Body(), &data); err != nil { return errors.New("wrong api call") } - res, err := c.be.CompleteMultipartUpload(bucket, "", uploadId, parts) + res, err := c.be.CompleteMultipartUpload(bucket, key, uploadId, data.Parts) return Responce(ctx, res, err) } res, err := c.be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key}) return Responce(ctx, res, err) } -func Responce[R comparable](ctx *fiber.Ctx, resp R, err error) error { +func Responce[R any](ctx *fiber.Ctx, resp R, err error) error { if err != nil { serr, ok := err.(s3err.APIError) if ok { @@ -414,6 +392,10 @@ func Responce[R comparable](ctx *fiber.Ctx, resp R, err error) error { return err } + if len(b) > 0 { + ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML) + } + return ctx.Send(b) } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index c27d6d5..48df055 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -29,6 +29,7 @@ import ( "github.com/valyala/fasthttp" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" ) func TestNew(t *testing.T) { @@ -128,8 +129,8 @@ func TestS3ApiController_GetActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{be: &BackendMock{ - ListObjectPartsFunc: func(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) { - return &s3.ListPartsOutput{}, nil + ListObjectPartsFunc: func(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { + return s3response.ListPartsResponse{}, nil }, GetObjectAclFunc: func(bucket, object string) (*s3.GetObjectAclOutput, error) { return &s3.GetObjectAclOutput{}, nil @@ -169,16 +170,16 @@ func TestS3ApiController_GetActions(t *testing.T) { req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=InvalidMaxParts", nil), }, wantErr: false, - statusCode: 500, + statusCode: 400, }, { - name: "Get-actions-invalid-part-number", + name: "Get-actions-invalid-part-number-marker", app: app, args: args{ req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=200&part-number-marker=InvalidPartNumber", nil), }, wantErr: false, - statusCode: 500, + statusCode: 400, }, { name: "Get-actions-list-object-parts-success", @@ -233,8 +234,8 @@ func TestS3ApiController_ListActions(t *testing.T) { GetBucketAclFunc: func(bucket string) (*s3.GetBucketAclOutput, error) { return &s3.GetBucketAclOutput{}, nil }, - ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) { - return &s3.ListMultipartUploadsOutput{}, nil + ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { + return s3response.ListMultipartUploadsResponse{}, nil }, ListObjectsV2Func: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { return &s3.ListObjectsV2Output{}, nil @@ -441,13 +442,13 @@ func TestS3ApiController_PutActions(t *testing.T) { statusCode int }{ { - name: "Upload-copy-part-error-case", + name: "Upload-put-part-error-case", app: app, args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?partNumber=invalid", nil), + req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=abc&partNumber=invalid", nil), }, wantErr: false, - statusCode: 500, + statusCode: 400, }, { name: "Upload-copy-part-success", @@ -517,11 +518,13 @@ func TestS3ApiController_PutActions(t *testing.T) { resp, err := tt.app.Test(tt.args.req) if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.GetActions() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("S3ApiController.GetActions() %v error = %v, wantErr %v", + tt.name, err, tt.wantErr) } if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.GetActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) + t.Errorf("S3ApiController.GetActions() %v statusCode = %v, wantStatusCode = %v", + tt.name, resp.StatusCode, tt.statusCode) } } } diff --git a/s3response/s3response.go b/s3response/s3response.go new file mode 100644 index 0000000..fec1033 --- /dev/null +++ b/s3response/s3response.go @@ -0,0 +1,96 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package s3response + +import ( + "encoding/xml" +) + +// Part describes part metadata. +type Part struct { + PartNumber int + LastModified string + ETag string + Size int64 +} + +// ListPartsResponse - s3 api list parts response. +type ListPartsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"` + + Bucket string + Key string + UploadID string `xml:"UploadId"` + + Initiator Initiator + Owner Owner + + // The class of storage used to store the object. + StorageClass string + + PartNumberMarker int + NextPartNumberMarker int + MaxParts int + IsTruncated bool + + // List of parts. + Parts []Part `xml:"Part"` +} + +// ListMultipartUploadsResponse - s3 api list multipart uploads response. +type ListMultipartUploadsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"` + + Bucket string + KeyMarker string + UploadIDMarker string `xml:"UploadIdMarker"` + NextKeyMarker string + NextUploadIDMarker string `xml:"NextUploadIdMarker"` + Delimiter string + Prefix string + EncodingType string `xml:"EncodingType,omitempty"` + MaxUploads int + IsTruncated bool + + // List of pending uploads. + Uploads []Upload `xml:"Upload"` + + // Delimed common prefixes. + CommonPrefixes []CommonPrefix +} + +// Upload desribes in progress multipart upload +type Upload struct { + Key string + UploadID string `xml:"UploadId"` + Initiator Initiator + Owner Owner + StorageClass string + Initiated string +} + +// CommonPrefix ListObjectsResponse common prefixes (directory abstraction) +type CommonPrefix struct { + Prefix string +} + +// Initiator same fields as Owner +type Initiator Owner + +// Owner bucket ownership +type Owner struct { + ID string + DisplayName string +}