mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-05 03:24:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement posix UploadCopyPart

Browse Source
This commit is contained in:
Ben McClelland
2023-06-30 21:09:44 -07:00
parent db484eb900
commit 47dea2db7c
4 changed files with 139 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
backend/backend.go
View File

@@ -39,11 +39,10 @@ type Backend interface {
CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(*s3.AbortMultipartUploadInput) error
ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error)
CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error)
PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error)
UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error)
UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
PutObject(*s3.PutObjectInput) (string, error)
HeadObject(bucket, object string) (*s3.HeadObjectOutput, error)
@@ -86,8 +85,8 @@ func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error {
func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketAcl(bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -117,9 +116,6 @@ func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsIn
func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}

118
backend/posix/posix.go
View File

@@ -693,10 +693,6 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke
}, nil
}
// TODO: copy part
// func (p *Posix) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) {
// }
func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -708,6 +704,15 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length
sum := sha256.Sum256([]byte(object))
objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err = os.Stat(filepath.Join(bucket, objdir, uploadID))
if errors.Is(err, fs.ErrNotExist) {
return "", s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if err != nil {
return "", fmt.Errorf("stat uploadid: %w", err)
}
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part))
f, err := openTmpFile(filepath.Join(bucket, objdir),
@@ -736,6 +741,111 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length
return etag, nil
}
func (p *Posix) UploadPartCopy(upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
_, err := os.Stat(*upi.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err)
}
sum := sha256.Sum256([]byte(*upi.Key))
objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err = os.Stat(filepath.Join(*upi.Bucket, objdir, *upi.UploadId))
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err)
}
partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", upi.PartNumber))
substrs := strings.SplitN(*upi.CopySource, "/", 2)
if len(substrs) != 2 {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidCopySource)
}
srcBucket := substrs[0]
srcObject := substrs[1]
_, err = os.Stat(srcBucket)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err)
}
objPath := filepath.Join(srcBucket, srcObject)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("stat object: %w", err)
}
startOffset, length, err := backend.ParseRange(fi, *upi.CopySourceRange)
if err != nil {
return s3response.CopyObjectResult{}, err
}
if length == -1 {
length = fi.Size() - startOffset + 1
}
if startOffset+length > fi.Size()+1 {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
f, err := openTmpFile(filepath.Join(*upi.Bucket, objdir),
*upi.Bucket, partPath, length)
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("open temp file: %w", err)
}
defer f.cleanup()
srcf, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("open object: %w", err)
}
defer srcf.Close()
rdr := io.NewSectionReader(srcf, startOffset, length)
hash := md5.New()
tr := io.TeeReader(rdr, hash)
_, err = io.Copy(f, tr)
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("copy part data: %w", err)
}
err = f.link()
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("link object in namespace: %w", err)
}
dataSum := hash.Sum(nil)
etag := hex.EncodeToString(dataSum)
xattr.Set(filepath.Join(*upi.Bucket, partPath), etagkey, []byte(etag))
fi, err = os.Stat(filepath.Join(*upi.Bucket, partPath))
if err != nil {
return s3response.CopyObjectResult{}, fmt.Errorf("stat part path: %w", err)
}
return s3response.CopyObjectResult{
ETag: etag,
LastModified: fi.ModTime(),
}, nil
}
func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
_, err := os.Stat(*po.Bucket)
if errors.Is(err, fs.ErrNotExist) {

112
s3api/controllers/backend_moq_test.go
View File

@@ -28,12 +28,9 @@ var _ backend.Backend = &BackendMock{}
// CompleteMultipartUploadFunc: func(bucket string, object string, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
// panic("mock out the CompleteMultipartUpload method")
// },
// CopyObjectFunc: func(srcBucket string, srcObject string, DstBucket string, dstObject string) (*s3.CopyObjectOutput, error) {
// CopyObjectFunc: func(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error) {
// panic("mock out the CopyObject method")
// },
// CopyPartFunc: func(srcBucket string, srcObject string, DstBucket string, uploadID string, rangeHeader string, part int) (*types.CopyPartResult, error) {
// panic("mock out the CopyPart method")
// },
// CreateMultipartUploadFunc: func(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
// panic("mock out the CreateMultipartUpload method")
// },
@@ -70,7 +67,7 @@ var _ backend.Backend = &BackendMock{}
// ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) {
// panic("mock out the ListBuckets method")
// },
// ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
// ListMultipartUploadsFunc: func(listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
// panic("mock out the ListMultipartUploads method")
// },
// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) {
@@ -112,7 +109,7 @@ var _ backend.Backend = &BackendMock{}
// StringFunc: func() string {
// panic("mock out the String method")
// },
// UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
// UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
// panic("mock out the UploadPartCopy method")
// },
// }
@@ -129,10 +126,7 @@ type BackendMock struct {
CompleteMultipartUploadFunc func(bucket string, object string, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error)
// CopyObjectFunc mocks the CopyObject method.
CopyObjectFunc func(srcBucket string, srcObject string, DstBucket string, dstObject string) (*s3.CopyObjectOutput, error)
// CopyPartFunc mocks the CopyPart method.
CopyPartFunc func(srcBucket string, srcObject string, DstBucket string, uploadID string, rangeHeader string, part int) (*types.CopyPartResult, error)
CopyObjectFunc func(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error)
// CreateMultipartUploadFunc mocks the CreateMultipartUpload method.
CreateMultipartUploadFunc func(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
@@ -171,7 +165,7 @@ type BackendMock struct {
ListBucketsFunc func() (s3response.ListAllMyBucketsResult, error)
// ListMultipartUploadsFunc mocks the ListMultipartUploads method.
ListMultipartUploadsFunc func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
ListMultipartUploadsFunc func(listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error)
// ListObjectPartsFunc mocks the ListObjectParts method.
ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error)
@@ -213,7 +207,7 @@ type BackendMock struct {
StringFunc func() string
// UploadPartCopyFunc mocks the UploadPartCopy method.
UploadPartCopyFunc func(uploadPartCopyInput *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error)
UploadPartCopyFunc func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
// calls tracks calls to the methods.
calls struct {
@@ -239,26 +233,11 @@ type BackendMock struct {
SrcBucket string
// SrcObject is the srcObject argument value.
SrcObject string
// DstBucket is the DstBucket argument value.
// DstBucket is the dstBucket argument value.
DstBucket string
// DstObject is the dstObject argument value.
DstObject string
}
// CopyPart holds details about calls to the CopyPart method.
CopyPart []struct {
// SrcBucket is the srcBucket argument value.
SrcBucket string
// SrcObject is the srcObject argument value.
SrcObject string
// DstBucket is the DstBucket argument value.
DstBucket string
// UploadID is the uploadID argument value.
UploadID string
// RangeHeader is the rangeHeader argument value.
RangeHeader string
// Part is the part argument value.
Part int
}
// CreateMultipartUpload holds details about calls to the CreateMultipartUpload method.
CreateMultipartUpload []struct {
// CreateMultipartUploadInput is the createMultipartUploadInput argument value.
@@ -339,8 +318,8 @@ type BackendMock struct {
}
// ListMultipartUploads holds details about calls to the ListMultipartUploads method.
ListMultipartUploads []struct {
// Output is the output argument value.
Output *s3.ListMultipartUploadsInput
// ListMultipartUploadsInput is the listMultipartUploadsInput argument value.
ListMultipartUploadsInput *s3.ListMultipartUploadsInput
}
// ListObjectParts holds details about calls to the ListObjectParts method.
ListObjectParts []struct {
@@ -460,7 +439,6 @@ type BackendMock struct {
lockAbortMultipartUpload sync.RWMutex
lockCompleteMultipartUpload sync.RWMutex
lockCopyObject sync.RWMutex
lockCopyPart sync.RWMutex
lockCreateMultipartUpload sync.RWMutex
lockDeleteBucket sync.RWMutex
lockDeleteObject sync.RWMutex
@@ -567,7 +545,7 @@ func (mock *BackendMock) CompleteMultipartUploadCalls() []struct {
}
// CopyObject calls CopyObjectFunc.
func (mock *BackendMock) CopyObject(srcBucket string, srcObject string, DstBucket string, dstObject string) (*s3.CopyObjectOutput, error) {
func (mock *BackendMock) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error) {
if mock.CopyObjectFunc == nil {
panic("BackendMock.CopyObjectFunc: method is nil but Backend.CopyObject was just called")
}
@@ -579,13 +557,13 @@ func (mock *BackendMock) CopyObject(srcBucket string, srcObject string, DstBucke
}{
SrcBucket: srcBucket,
SrcObject: srcObject,
DstBucket: DstBucket,
DstBucket: dstBucket,
DstObject: dstObject,
}
mock.lockCopyObject.Lock()
mock.calls.CopyObject = append(mock.calls.CopyObject, callInfo)
mock.lockCopyObject.Unlock()
return mock.CopyObjectFunc(srcBucket, srcObject, DstBucket, dstObject)
return mock.CopyObjectFunc(srcBucket, srcObject, dstBucket, dstObject)
}
// CopyObjectCalls gets all the calls that were made to CopyObject.
@@ -610,58 +588,6 @@ func (mock *BackendMock) CopyObjectCalls() []struct {
return calls
}
// CopyPart calls CopyPartFunc.
func (mock *BackendMock) CopyPart(srcBucket string, srcObject string, DstBucket string, uploadID string, rangeHeader string, part int) (*types.CopyPartResult, error) {
if mock.CopyPartFunc == nil {
panic("BackendMock.CopyPartFunc: method is nil but Backend.CopyPart was just called")
}
callInfo := struct {
SrcBucket string
SrcObject string
DstBucket string
UploadID string
RangeHeader string
Part int
}{
SrcBucket: srcBucket,
SrcObject: srcObject,
DstBucket: DstBucket,
UploadID: uploadID,
RangeHeader: rangeHeader,
Part: part,
}
mock.lockCopyPart.Lock()
mock.calls.CopyPart = append(mock.calls.CopyPart, callInfo)
mock.lockCopyPart.Unlock()
return mock.CopyPartFunc(srcBucket, srcObject, DstBucket, uploadID, rangeHeader, part)
}
// CopyPartCalls gets all the calls that were made to CopyPart.
// Check the length with:
//
// len(mockedBackend.CopyPartCalls())
func (mock *BackendMock) CopyPartCalls() []struct {
SrcBucket string
SrcObject string
DstBucket string
UploadID string
RangeHeader string
Part int
} {
var calls []struct {
SrcBucket string
SrcObject string
DstBucket string
UploadID string
RangeHeader string
Part int
}
mock.lockCopyPart.RLock()
calls = mock.calls.CopyPart
mock.lockCopyPart.RUnlock()
return calls
}
// CreateMultipartUpload calls CreateMultipartUploadFunc.
func (mock *BackendMock) CreateMultipartUpload(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
if mock.CreateMultipartUploadFunc == nil {
@@ -1082,19 +1008,19 @@ func (mock *BackendMock) ListBucketsCalls() []struct {
}
// ListMultipartUploads calls ListMultipartUploadsFunc.
func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
func (mock *BackendMock) ListMultipartUploads(listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) {
if mock.ListMultipartUploadsFunc == nil {
panic("BackendMock.ListMultipartUploadsFunc: method is nil but Backend.ListMultipartUploads was just called")
}
callInfo := struct {
Output *s3.ListMultipartUploadsInput
ListMultipartUploadsInput *s3.ListMultipartUploadsInput
}{
Output: output,
ListMultipartUploadsInput: listMultipartUploadsInput,
}
mock.lockListMultipartUploads.Lock()
mock.calls.ListMultipartUploads = append(mock.calls.ListMultipartUploads, callInfo)
mock.lockListMultipartUploads.Unlock()
return mock.ListMultipartUploadsFunc(output)
return mock.ListMultipartUploadsFunc(listMultipartUploadsInput)
}
// ListMultipartUploadsCalls gets all the calls that were made to ListMultipartUploads.
@@ -1102,10 +1028,10 @@ func (mock *BackendMock) ListMultipartUploads(output *s3.ListMultipartUploadsInp
//
// len(mockedBackend.ListMultipartUploadsCalls())
func (mock *BackendMock) ListMultipartUploadsCalls() []struct {
Output *s3.ListMultipartUploadsInput
ListMultipartUploadsInput *s3.ListMultipartUploadsInput
} {
var calls []struct {
Output *s3.ListMultipartUploadsInput
ListMultipartUploadsInput *s3.ListMultipartUploadsInput
}
mock.lockListMultipartUploads.RLock()
calls = mock.calls.ListMultipartUploads
@@ -1616,7 +1542,7 @@ func (mock *BackendMock) StringCalls() []struct {
}
// UploadPartCopy calls UploadPartCopyFunc.
func (mock *BackendMock) UploadPartCopy(uploadPartCopyInput *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
func (mock *BackendMock) UploadPartCopy(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
if mock.UploadPartCopyFunc == nil {
panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called")
}

4
s3api/controllers/base_test.go
View File

@@ -667,8 +667,8 @@ func TestS3ApiController_PutActions(t *testing.T) {
SetTagsFunc: func(bucket, object string, tags map[string]string) error {
return nil
},
UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
return &s3.UploadPartCopyOutput{}, nil
UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, nil
},
},
}