mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-05 03:24:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #21 from versity/issue-12

Browse Source 
Issue 12
This commit is contained in:
Ben McClelland
2023-05-24 08:24:40 -07:00
committed by GitHub
parent 50b0e454e6 dcdc62411e
commit b41dfd653c
7 changed files with 71 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/backend.go
View File

@@ -30,7 +30,7 @@ type Backend interface {
CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error)
PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error)
PutObject(bucket, object string, r io.Reader) (string, error)
PutObject(*s3.PutObjectInput) (string, error)
HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error)
GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3.GetObjectOutput, error)
GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
@@ -120,7 +120,7 @@ func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part in
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(bucket, object string, r io.Reader) (string, error) {
func (BackendUnsupported) PutObject(*s3.PutObjectInput) (string, error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(bucket, object string) error {

32
backend/backend_moq_test.go
View File

@@ -92,7 +92,7 @@ var _ Backend = &BackendMock{}
// PutBucketAclFunc: func(putBucketAclInput *s3.PutBucketAclInput) error {
// panic("mock out the PutBucketAcl method")
// },
// PutObjectFunc: func(bucket string, object string, r io.Reader) (string, error) {
// PutObjectFunc: func(putObjectInput *s3.PutObjectInput) (string, error) {
// panic("mock out the PutObject method")
// },
// PutObjectAclFunc: func(putObjectAclInput *s3.PutObjectAclInput) error {
@@ -202,7 +202,7 @@ type BackendMock struct {
PutBucketAclFunc func(putBucketAclInput *s3.PutBucketAclInput) error
// PutObjectFunc mocks the PutObject method.
PutObjectFunc func(bucket string, object string, r io.Reader) (string, error)
PutObjectFunc func(putObjectInput *s3.PutObjectInput) (string, error)
// PutObjectAclFunc mocks the PutObjectAcl method.
PutObjectAclFunc func(putObjectAclInput *s3.PutObjectAclInput) error
@@ -421,12 +421,8 @@ type BackendMock struct {
}
// PutObject holds details about calls to the PutObject method.
PutObject []struct {
// Bucket is the bucket argument value.
Bucket string
// Object is the object argument value.
Object string
// R is the r argument value.
R io.Reader
// PutObjectInput is the putObjectInput argument value.
PutObjectInput *s3.PutObjectInput
}
// PutObjectAcl holds details about calls to the PutObjectAcl method.
PutObjectAcl []struct {
@@ -1430,23 +1426,19 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
}
// PutObject calls PutObjectFunc.
func (mock *BackendMock) PutObject(bucket string, object string, r io.Reader) (string, error) {
func (mock *BackendMock) PutObject(putObjectInput *s3.PutObjectInput) (string, error) {
if mock.PutObjectFunc == nil {
panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called")
}
callInfo := struct {
Bucket string
Object string
R io.Reader
PutObjectInput *s3.PutObjectInput
}{
Bucket: bucket,
Object: object,
R: r,
PutObjectInput: putObjectInput,
}
mock.lockPutObject.Lock()
mock.calls.PutObject = append(mock.calls.PutObject, callInfo)
mock.lockPutObject.Unlock()
return mock.PutObjectFunc(bucket, object, r)
return mock.PutObjectFunc(putObjectInput)
}
// PutObjectCalls gets all the calls that were made to PutObject.
@@ -1454,14 +1446,10 @@ func (mock *BackendMock) PutObject(bucket string, object string, r io.Reader) (s
//
// len(mockedBackend.PutObjectCalls())
func (mock *BackendMock) PutObjectCalls() []struct {
Bucket string
Object string
R io.Reader
PutObjectInput *s3.PutObjectInput
} {
var calls []struct {
Bucket string
Object string
R io.Reader
PutObjectInput *s3.PutObjectInput
}
mock.lockPutObject.RLock()
calls = mock.calls.PutObject

16
backend/posix/posix.go
View File

@@ -682,8 +682,8 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length
return etag, nil
}
func (p *Posix) PutObject(bucket, object string, r io.Reader) (string, error) {
_, err := os.Stat(bucket)
func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
_, err := os.Stat(*po.Bucket)
if err != nil && os.IsNotExist(err) {
return "", s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
@@ -691,13 +691,13 @@ func (p *Posix) PutObject(bucket, object string, r io.Reader) (string, error) {
return "", fmt.Errorf("stat bucket: %w", err)
}
name := filepath.Join(bucket, object)
name := filepath.Join(*po.Bucket, *po.Key)
etag := ""
if strings.HasSuffix(object, "/") {
if strings.HasSuffix(*po.Key, "/") {
// object is directory
err = mkdirAll(name, os.FileMode(0755), bucket, object)
err = mkdirAll(name, os.FileMode(0755), *po.Bucket, *po.Key)
if err != nil {
return "", err
}
@@ -720,7 +720,7 @@ func (p *Posix) PutObject(bucket, object string, r io.Reader) (string, error) {
// TODO: fallocate based on content length
hash := md5.New()
rdr := io.TeeReader(r, hash)
rdr := io.TeeReader(po.Body, hash)
_, err = io.Copy(f, rdr)
if err != nil {
f.Close()
@@ -728,7 +728,7 @@ func (p *Posix) PutObject(bucket, object string, r io.Reader) (string, error) {
}
dir := filepath.Dir(name)
if dir != "" {
err = mkdirAll(dir, os.FileMode(0755), bucket, object)
err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key)
if err != nil {
f.Close()
return "", fmt.Errorf("make object parent directories: %w", err)
@@ -936,7 +936,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
}
defer f.Close()
etag, err := p.PutObject(DstBucket, dstObject, f)
etag, err := p.PutObject(&s3.PutObjectInput{Bucket: &DstBucket, Key: &dstObject, Body: f})
if err != nil {
return nil, err
}

32
s3api/controllers/backend_moq_test.go
View File

@@ -93,7 +93,7 @@ var _ backend.Backend = &BackendMock{}
// PutBucketAclFunc: func(putBucketAclInput *s3.PutBucketAclInput) error {
// panic("mock out the PutBucketAcl method")
// },
// PutObjectFunc: func(bucket string, object string, r io.Reader) (string, error) {
// PutObjectFunc: func(putObjectInput *s3.PutObjectInput) (string, error) {
// panic("mock out the PutObject method")
// },
// PutObjectAclFunc: func(putObjectAclInput *s3.PutObjectAclInput) error {
@@ -203,7 +203,7 @@ type BackendMock struct {
PutBucketAclFunc func(putBucketAclInput *s3.PutBucketAclInput) error
// PutObjectFunc mocks the PutObject method.
PutObjectFunc func(bucket string, object string, r io.Reader) (string, error)
PutObjectFunc func(putObjectInput *s3.PutObjectInput) (string, error)
// PutObjectAclFunc mocks the PutObjectAcl method.
PutObjectAclFunc func(putObjectAclInput *s3.PutObjectAclInput) error
@@ -422,12 +422,8 @@ type BackendMock struct {
}
// PutObject holds details about calls to the PutObject method.
PutObject []struct {
// Bucket is the bucket argument value.
Bucket string
// Object is the object argument value.
Object string
// R is the r argument value.
R io.Reader
// PutObjectInput is the putObjectInput argument value.
PutObjectInput *s3.PutObjectInput
}
// PutObjectAcl holds details about calls to the PutObjectAcl method.
PutObjectAcl []struct {
@@ -1431,23 +1427,19 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
}
// PutObject calls PutObjectFunc.
func (mock *BackendMock) PutObject(bucket string, object string, r io.Reader) (string, error) {
func (mock *BackendMock) PutObject(putObjectInput *s3.PutObjectInput) (string, error) {
if mock.PutObjectFunc == nil {
panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called")
}
callInfo := struct {
Bucket string
Object string
R io.Reader
PutObjectInput *s3.PutObjectInput
}{
Bucket: bucket,
Object: object,
R: r,
PutObjectInput: putObjectInput,
}
mock.lockPutObject.Lock()
mock.calls.PutObject = append(mock.calls.PutObject, callInfo)
mock.lockPutObject.Unlock()
return mock.PutObjectFunc(bucket, object, r)
return mock.PutObjectFunc(putObjectInput)
}
// PutObjectCalls gets all the calls that were made to PutObject.
@@ -1455,14 +1447,10 @@ func (mock *BackendMock) PutObject(bucket string, object string, r io.Reader) (s
//
// len(mockedBackend.PutObjectCalls())
func (mock *BackendMock) PutObjectCalls() []struct {
Bucket string
Object string
R io.Reader
PutObjectInput *s3.PutObjectInput
} {
var calls []struct {
Bucket string
Object string
R io.Reader
PutObjectInput *s3.PutObjectInput
}
mock.lockPutObject.RLock()
calls = mock.calls.PutObject

22
s3api/controllers/base.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/versity/scoutgw/backend"
"github.com/versity/scoutgw/s3api/utils"
"github.com/versity/scoutgw/s3err"
)
@@ -143,7 +144,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
copySource, copySrcIfMatch, copySrcIfNoneMatch,
copySrcModifSince, copySrcUnmodifSince, acl,
grantFullControl, grantRead, grantReadACP,
granWrite, grantWriteACP :=
granWrite, grantWriteACP, contentLengthStr :=
// Copy source headers
ctx.Get("X-Amz-Copy-Source"),
ctx.Get("X-Amz-Copy-Source-If-Match"),
@@ -156,7 +157,9 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ctx.Get("X-Amz-Grant-Read"),
ctx.Get("X-Amz-Grant-Read-Acp"),
ctx.Get("X-Amz-Grant-Write"),
ctx.Get("X-Amz-Grant-Write-Acp")
ctx.Get("X-Amz-Grant-Write-Acp"),
// Other headers
ctx.Get("Content-Length")
grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP
@@ -227,7 +230,20 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return responce(ctx, res, err)
}
res, err := c.be.PutObject(dstBucket, dstKeyStart, bytes.NewReader(ctx.Request().Body()))
contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return errors.New("wrong api call")
}
metadata := utils.GetUserMetaData(&ctx.Request().Header)
res, err := c.be.PutObject(&s3.PutObjectInput{
Bucket: &dstBucket,
Key: &dstKeyStart,
ContentLength: contentLength,
Metadata: metadata,
Body: bytes.NewReader(ctx.Request().Body()),
})
return responce(ctx, res, err)
}

4
s3api/controllers/base_test.go
View File

@@ -406,8 +406,8 @@ func TestS3ApiController_PutActions(t *testing.T) {
CopyObjectFunc: func(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
return &s3.CopyObjectOutput{}, nil
},
PutObjectFunc: func(bucket, object string, r io.Reader) (string, error) {
return "hello", nil
PutObjectFunc: func(*s3.PutObjectInput) (string, error) {
return "Hey", nil
},
}}
app.Put("/:bucket/:key/*", s3ApiController.PutActions)

20
s3api/utils/utils.go Normal file
View File

@@ -0,0 +1,20 @@
package utils
import (
"strings"
"github.com/valyala/fasthttp"
)
func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
metadata = make(map[string]string)
headers.VisitAll(func(key, value []byte) {
if strings.HasPrefix(string(key), "X-Amz-Meta-") {
trimmedKey := strings.TrimPrefix(string(key), "X-Amz-Meta-")
headerValue := string(value)
metadata[trimmedKey] = headerValue
}
})
return
}