diff --git a/backend/backend.go b/backend/backend.go index c8f4e3c..71e9654 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -39,11 +39,10 @@ type Backend interface { CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) AbortMultipartUpload(*s3.AbortMultipartUploadInput) error - ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) + ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) - CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) - UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) + UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) PutObject(*s3.PutObjectInput) (string, error) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) @@ -86,8 +85,8 @@ func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error { func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) GetBucketAcl(bucket string) ([]byte, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) @@ -117,9 +116,6 @@ func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsIn func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) -} func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) { return "", s3err.GetAPIError(s3err.ErrNotImplemented) } diff --git a/backend/posix/posix.go b/backend/posix/posix.go index ff587b6..4729e9e 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -693,10 +693,6 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke }, nil } -// TODO: copy part -// func (p *Posix) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) { -// } - func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) { _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -708,6 +704,15 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length sum := sha256.Sum256([]byte(object)) objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + + _, err = os.Stat(filepath.Join(bucket, objdir, uploadID)) + if errors.Is(err, fs.ErrNotExist) { + return "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + if err != nil { + return "", fmt.Errorf("stat uploadid: %w", err) + } + partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part)) f, err := openTmpFile(filepath.Join(bucket, objdir), @@ -736,6 +741,111 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length return etag, nil } +func (p *Posix) UploadPartCopy(upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { + _, err := os.Stat(*upi.Bucket) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err) + } + + sum := sha256.Sum256([]byte(*upi.Key)) + objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + + _, err = os.Stat(filepath.Join(*upi.Bucket, objdir, *upi.UploadId)) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err) + } + + partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", upi.PartNumber)) + + substrs := strings.SplitN(*upi.CopySource, "/", 2) + if len(substrs) != 2 { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + + srcBucket := substrs[0] + srcObject := substrs[1] + + _, err = os.Stat(srcBucket) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err) + } + + objPath := filepath.Join(srcBucket, srcObject) + fi, err := os.Stat(objPath) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("stat object: %w", err) + } + + startOffset, length, err := backend.ParseRange(fi, *upi.CopySourceRange) + if err != nil { + return s3response.CopyObjectResult{}, err + } + + if length == -1 { + length = fi.Size() - startOffset + 1 + } + + if startOffset+length > fi.Size()+1 { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + f, err := openTmpFile(filepath.Join(*upi.Bucket, objdir), + *upi.Bucket, partPath, length) + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("open temp file: %w", err) + } + defer f.cleanup() + + srcf, err := os.Open(objPath) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("open object: %w", err) + } + defer srcf.Close() + + rdr := io.NewSectionReader(srcf, startOffset, length) + hash := md5.New() + tr := io.TeeReader(rdr, hash) + + _, err = io.Copy(f, tr) + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("copy part data: %w", err) + } + + err = f.link() + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("link object in namespace: %w", err) + } + + dataSum := hash.Sum(nil) + etag := hex.EncodeToString(dataSum) + xattr.Set(filepath.Join(*upi.Bucket, partPath), etagkey, []byte(etag)) + + fi, err = os.Stat(filepath.Join(*upi.Bucket, partPath)) + if err != nil { + return s3response.CopyObjectResult{}, fmt.Errorf("stat part path: %w", err) + } + + return s3response.CopyObjectResult{ + ETag: etag, + LastModified: fi.ModTime(), + }, nil +} + func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { _, err := os.Stat(*po.Bucket) if errors.Is(err, fs.ErrNotExist) { diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 39adc7a..03d6420 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -28,12 +28,9 @@ var _ backend.Backend = &BackendMock{} // CompleteMultipartUploadFunc: func(bucket string, object string, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) { // panic("mock out the CompleteMultipartUpload method") // }, -// CopyObjectFunc: func(srcBucket string, srcObject string, DstBucket string, dstObject string) (*s3.CopyObjectOutput, error) { +// CopyObjectFunc: func(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error) { // panic("mock out the CopyObject method") // }, -// CopyPartFunc: func(srcBucket string, srcObject string, DstBucket string, uploadID string, rangeHeader string, part int) (*types.CopyPartResult, error) { -// panic("mock out the CopyPart method") -// }, // CreateMultipartUploadFunc: func(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { // panic("mock out the CreateMultipartUpload method") // }, @@ -70,7 +67,7 @@ var _ backend.Backend = &BackendMock{} // ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) { // panic("mock out the ListBuckets method") // }, -// ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { +// ListMultipartUploadsFunc: func(listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { // panic("mock out the ListMultipartUploads method") // }, // ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { @@ -112,7 +109,7 @@ var _ backend.Backend = &BackendMock{} // StringFunc: func() string { // panic("mock out the String method") // }, -// UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) { +// UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { // panic("mock out the UploadPartCopy method") // }, // } @@ -129,10 +126,7 @@ type BackendMock struct { CompleteMultipartUploadFunc func(bucket string, object string, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) // CopyObjectFunc mocks the CopyObject method. - CopyObjectFunc func(srcBucket string, srcObject string, DstBucket string, dstObject string) (*s3.CopyObjectOutput, error) - - // CopyPartFunc mocks the CopyPart method. - CopyPartFunc func(srcBucket string, srcObject string, DstBucket string, uploadID string, rangeHeader string, part int) (*types.CopyPartResult, error) + CopyObjectFunc func(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error) // CreateMultipartUploadFunc mocks the CreateMultipartUpload method. CreateMultipartUploadFunc func(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) @@ -171,7 +165,7 @@ type BackendMock struct { ListBucketsFunc func() (s3response.ListAllMyBucketsResult, error) // ListMultipartUploadsFunc mocks the ListMultipartUploads method. - ListMultipartUploadsFunc func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) + ListMultipartUploadsFunc func(listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) // ListObjectPartsFunc mocks the ListObjectParts method. ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) @@ -213,7 +207,7 @@ type BackendMock struct { StringFunc func() string // UploadPartCopyFunc mocks the UploadPartCopy method. - UploadPartCopyFunc func(uploadPartCopyInput *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) + UploadPartCopyFunc func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) // calls tracks calls to the methods. calls struct { @@ -239,26 +233,11 @@ type BackendMock struct { SrcBucket string // SrcObject is the srcObject argument value. SrcObject string - // DstBucket is the DstBucket argument value. + // DstBucket is the dstBucket argument value. DstBucket string // DstObject is the dstObject argument value. DstObject string } - // CopyPart holds details about calls to the CopyPart method. - CopyPart []struct { - // SrcBucket is the srcBucket argument value. - SrcBucket string - // SrcObject is the srcObject argument value. - SrcObject string - // DstBucket is the DstBucket argument value. - DstBucket string - // UploadID is the uploadID argument value. - UploadID string - // RangeHeader is the rangeHeader argument value. - RangeHeader string - // Part is the part argument value. - Part int - } // CreateMultipartUpload holds details about calls to the CreateMultipartUpload method. CreateMultipartUpload []struct { // CreateMultipartUploadInput is the createMultipartUploadInput argument value. @@ -339,8 +318,8 @@ type BackendMock struct { } // ListMultipartUploads holds details about calls to the ListMultipartUploads method. ListMultipartUploads []struct { - // Output is the output argument value. - Output *s3.ListMultipartUploadsInput + // ListMultipartUploadsInput is the listMultipartUploadsInput argument value. + ListMultipartUploadsInput *s3.ListMultipartUploadsInput } // ListObjectParts holds details about calls to the ListObjectParts method. ListObjectParts []struct { @@ -460,7 +439,6 @@ type BackendMock struct { lockAbortMultipartUpload sync.RWMutex lockCompleteMultipartUpload sync.RWMutex lockCopyObject sync.RWMutex - lockCopyPart sync.RWMutex lockCreateMultipartUpload sync.RWMutex lockDeleteBucket sync.RWMutex lockDeleteObject sync.RWMutex @@ -567,7 +545,7 @@ func (mock *BackendMock) CompleteMultipartUploadCalls() []struct { } // CopyObject calls CopyObjectFunc. -func (mock *BackendMock) CopyObject(srcBucket string, srcObject string, DstBucket string, dstObject string) (*s3.CopyObjectOutput, error) { +func (mock *BackendMock) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error) { if mock.CopyObjectFunc == nil { panic("BackendMock.CopyObjectFunc: method is nil but Backend.CopyObject was just called") } @@ -579,13 +557,13 @@ func (mock *BackendMock) CopyObject(srcBucket string, srcObject string, DstBucke }{ SrcBucket: srcBucket, SrcObject: srcObject, - DstBucket: DstBucket, + DstBucket: dstBucket, DstObject: dstObject, } mock.lockCopyObject.Lock() mock.calls.CopyObject = append(mock.calls.CopyObject, callInfo) mock.lockCopyObject.Unlock() - return mock.CopyObjectFunc(srcBucket, srcObject, DstBucket, dstObject) + return mock.CopyObjectFunc(srcBucket, srcObject, dstBucket, dstObject) } // CopyObjectCalls gets all the calls that were made to CopyObject. @@ -610,58 +588,6 @@ func (mock *BackendMock) CopyObjectCalls() []struct { return calls } -// CopyPart calls CopyPartFunc. -func (mock *BackendMock) CopyPart(srcBucket string, srcObject string, DstBucket string, uploadID string, rangeHeader string, part int) (*types.CopyPartResult, error) { - if mock.CopyPartFunc == nil { - panic("BackendMock.CopyPartFunc: method is nil but Backend.CopyPart was just called") - } - callInfo := struct { - SrcBucket string - SrcObject string - DstBucket string - UploadID string - RangeHeader string - Part int - }{ - SrcBucket: srcBucket, - SrcObject: srcObject, - DstBucket: DstBucket, - UploadID: uploadID, - RangeHeader: rangeHeader, - Part: part, - } - mock.lockCopyPart.Lock() - mock.calls.CopyPart = append(mock.calls.CopyPart, callInfo) - mock.lockCopyPart.Unlock() - return mock.CopyPartFunc(srcBucket, srcObject, DstBucket, uploadID, rangeHeader, part) -} - -// CopyPartCalls gets all the calls that were made to CopyPart. -// Check the length with: -// -// len(mockedBackend.CopyPartCalls()) -func (mock *BackendMock) CopyPartCalls() []struct { - SrcBucket string - SrcObject string - DstBucket string - UploadID string - RangeHeader string - Part int -} { - var calls []struct { - SrcBucket string - SrcObject string - DstBucket string - UploadID string - RangeHeader string - Part int - } - mock.lockCopyPart.RLock() - calls = mock.calls.CopyPart - mock.lockCopyPart.RUnlock() - return calls -} - // CreateMultipartUpload calls CreateMultipartUploadFunc. func (mock *BackendMock) CreateMultipartUpload(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { if mock.CreateMultipartUploadFunc == nil { @@ -1082,19 +1008,19 @@ func (mock *BackendMock) ListBucketsCalls() []struct { } // ListMultipartUploads calls ListMultipartUploadsFunc. -func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { +func (mock *BackendMock) ListMultipartUploads(listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { if mock.ListMultipartUploadsFunc == nil { panic("BackendMock.ListMultipartUploadsFunc: method is nil but Backend.ListMultipartUploads was just called") } callInfo := struct { - Output *s3.ListMultipartUploadsInput + ListMultipartUploadsInput *s3.ListMultipartUploadsInput }{ - Output: output, + ListMultipartUploadsInput: listMultipartUploadsInput, } mock.lockListMultipartUploads.Lock() mock.calls.ListMultipartUploads = append(mock.calls.ListMultipartUploads, callInfo) mock.lockListMultipartUploads.Unlock() - return mock.ListMultipartUploadsFunc(output) + return mock.ListMultipartUploadsFunc(listMultipartUploadsInput) } // ListMultipartUploadsCalls gets all the calls that were made to ListMultipartUploads. @@ -1102,10 +1028,10 @@ func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInp // // len(mockedBackend.ListMultipartUploadsCalls()) func (mock *BackendMock) ListMultipartUploadsCalls() []struct { - Output *s3.ListMultipartUploadsInput + ListMultipartUploadsInput *s3.ListMultipartUploadsInput } { var calls []struct { - Output *s3.ListMultipartUploadsInput + ListMultipartUploadsInput *s3.ListMultipartUploadsInput } mock.lockListMultipartUploads.RLock() calls = mock.calls.ListMultipartUploads @@ -1616,7 +1542,7 @@ func (mock *BackendMock) StringCalls() []struct { } // UploadPartCopy calls UploadPartCopyFunc. -func (mock *BackendMock) UploadPartCopy(uploadPartCopyInput *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) { +func (mock *BackendMock) UploadPartCopy(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { if mock.UploadPartCopyFunc == nil { panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called") } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 0a28d75..ba98481 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -667,8 +667,8 @@ func TestS3ApiController_PutActions(t *testing.T) { SetTagsFunc: func(bucket, object string, tags map[string]string) error { return nil }, - UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) { - return &s3.UploadPartCopyOutput{}, nil + UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { + return s3response.CopyObjectResult{}, nil }, }, }