mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-20 15:02:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Closes #290, implemented request body stream reading for PutObject and UploadPart actions.

Browse Source
This commit is contained in:
jonaustin09
2023-11-15 12:47:45 -05:00
committed by Ben McClelland
parent 90bb43f7c9
commit 27eb43d089
7 changed files with 134 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/posix/posix.go
+11
View File
@@ -37,6 +37,7 @@ import (
"github.com/pkg/xattr"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
@@ -894,6 +895,11 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
return "", fmt.Errorf("write part data: %w", err)
}
rdr, ok := r.(*utils.HashReader)
if ok && rdr.Err() != nil {
return "", err
}
err = f.link()
if err != nil {
return "", fmt.Errorf("link object in namespace: %w", err)
@@ -1101,6 +1107,11 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
if err != nil {
return "", fmt.Errorf("write object data: %w", err)
}
r, ok := po.Body.(*utils.HashReader)
if ok && r.Err() != nil {
return "", r.Err()
}
dir := filepath.Dir(name)
if dir != "" {
err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key)
cmd/versitygw/main.go
+4 -3
View File
@@ -289,9 +289,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
}
app := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
BodyLimit: int(blimit),
AppName: "versitygw",
ServerHeader: "VERSITYGW",
BodyLimit: int(blimit),
StreamRequestBody: true,
})
var opts []s3api.Option
s3api/controllers/base.go
+4 -3
View File
@@ -15,7 +15,6 @@
package controllers
import (
"bytes"
"encoding/xml"
"fmt"
"io"
@@ -516,7 +515,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
}
body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
body := ctx.Locals("body-reader").(io.Reader)
ctx.Locals("logReqBody", false)
etag, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{
Bucket: &bucket,
@@ -655,13 +654,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner})
}
rdr := ctx.Locals("body-reader").(io.Reader)
ctx.Locals("logReqBody", false)
etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{
Bucket: &bucket,
Key: &keyStart,
ContentLength: &contentLength,
Metadata: metadata,
Body: bytes.NewReader(ctx.Request().Body()),
Body: rdr,
Tagging: &tagging,
})
ctx.Response().Header.Set("ETag", etag)
s3api/middlewares/authentication.go
+14 -7
View File
@@ -143,13 +143,19 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
ok := isSpecialPayload(hashPayloadHeader)
if !ok {
// Calculate the hash of the request payload
hashedPayload := sha256.Sum256(ctx.Body())
hexPayload := hex.EncodeToString(hashedPayload[:])
if utils.IsBigDataAction(ctx) {
rdr := ctx.Request().BodyStream()
r := utils.NewHashReader(rdr, sha256.New(), hashPayloadHeader, utils.HashTypeSha256)
ctx.Locals("body-reader", r)
} else {
// Calculate the hash of the request payload
hashedPayload := sha256.Sum256(ctx.Body())
hexPayload := hex.EncodeToString(hashedPayload[:])
// Compare the calculated hash with the hash provided
if hashPayloadHeader != hexPayload {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
// Compare the calculated hash with the hash provided
if hashPayloadHeader != hexPayload {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
}
}
}
@@ -166,7 +172,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
SecretAccessKey: account.Secret,
}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
options.DisableURIPathEscaping = true
if debug {
if true {
options.LogSigning = true
options.Logger = logging.NewStandardLogger(os.Stderr)
}
@@ -181,6 +187,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
}
calculatedSign := strings.Split(parts[3], "=")[1]
expectedSign := strings.Split(authParts[2], "=")[1]
fmt.Println(calculatedSign, expectedSign)
if expectedSign != calculatedSign {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger})
s3api/middlewares/md5.go
+11 -4
View File
@@ -20,6 +20,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
@@ -31,11 +32,17 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
return ctx.Next()
}
sum := md5.Sum(ctx.Body())
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
if utils.IsBigDataAction(ctx) {
rdr := ctx.Locals("body-reader").(*utils.HashReader)
r := utils.NewHashReader(rdr, md5.New(), incomingSum, utils.HashTypeMd5)
ctx.Locals("body-reader", r)
} else {
sum := md5.Sum(ctx.Body())
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
}
}
return ctx.Next()
s3api/utils/reader.go
+72
View File
@@ -0,0 +1,72 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"encoding/base64"
"encoding/hex"
"errors"
"hash"
"io"
"github.com/versity/versitygw/s3err"
)
type HashType string
const (
HashTypeMd5 = "md5"
HashTypeSha256 = "sha256"
)
type HashReader struct {
hashType HashType
hash hash.Hash
r io.Reader
sum string
err error
}
func NewHashReader(r io.Reader, hash hash.Hash, sum string, ht HashType) *HashReader {
return &HashReader{hash: hash, r: r, sum: sum, hashType: ht}
}
func (hr *HashReader) Read(p []byte) (int, error) {
n, readerr := hr.r.Read(p)
_, err := hr.hash.Write(p[:n])
if err != nil {
return n, err
}
if errors.Is(readerr, io.EOF) {
if hr.hashType == HashTypeMd5 {
sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
if sum != hr.sum {
hr.err = s3err.GetAPIError(s3err.ErrInvalidDigest)
return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
}
} else if hr.hashType == HashTypeSha256 {
sum := hex.EncodeToString(hr.hash.Sum(nil))
if sum != hr.sum {
hr.err = s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
}
}
}
return n, readerr
}
func (hr *HashReader) Err() error {
return hr.err
}
s3api/utils/utils.go
+18 -1
View File
@@ -18,6 +18,7 @@ import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"regexp"
"strconv"
@@ -51,8 +52,15 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin
func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
req := ctx.Request()
var body io.Reader
if IsBigDataAction(ctx) {
body = req.BodyStream()
fmt.Println("create ctx body: ", body)
} else {
body = bytes.NewReader(req.Body())
}
httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), bytes.NewReader(req.Body()))
httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
if err != nil {
return nil, errors.New("error in creating an http request")
}
@@ -131,3 +139,12 @@ func includeHeader(hdr string, signedHdrs []string) bool {
}
return false
}
func IsBigDataAction(ctx *fiber.Ctx) bool {
if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
return true
}
}
return false
}