diff --git a/backend/posix/posix.go b/backend/posix/posix.go index a5c88da7..f40af9bd 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -37,6 +37,7 @@ import ( "github.com/pkg/xattr" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -894,6 +895,11 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string return "", fmt.Errorf("write part data: %w", err) } + rdr, ok := r.(*utils.HashReader) + if ok && rdr.Err() != nil { + return "", err + } + err = f.link() if err != nil { return "", fmt.Errorf("link object in namespace: %w", err) @@ -1101,6 +1107,11 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e if err != nil { return "", fmt.Errorf("write object data: %w", err) } + + r, ok := po.Body.(*utils.HashReader) + if ok && r.Err() != nil { + return "", r.Err() + } dir := filepath.Dir(name) if dir != "" { err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key) diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 79d03719..33a308ce 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -289,9 +289,10 @@ func runGateway(ctx context.Context, be backend.Backend) error { } app := fiber.New(fiber.Config{ - AppName: "versitygw", - ServerHeader: "VERSITYGW", - BodyLimit: int(blimit), + AppName: "versitygw", + ServerHeader: "VERSITYGW", + BodyLimit: int(blimit), + StreamRequestBody: true, }) var opts []s3api.Option diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 01766643..d5a5b32b 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -15,7 +15,6 @@ package controllers import ( - "bytes" "encoding/xml" "fmt" "io" @@ -516,7 +515,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner}) } - body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body()))) + body := ctx.Locals("body-reader").(io.Reader) ctx.Locals("logReqBody", false) etag, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{ Bucket: &bucket, @@ -655,13 +654,15 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner}) } + rdr := ctx.Locals("body-reader").(io.Reader) + ctx.Locals("logReqBody", false) etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{ Bucket: &bucket, Key: &keyStart, ContentLength: &contentLength, Metadata: metadata, - Body: bytes.NewReader(ctx.Request().Body()), + Body: rdr, Tagging: &tagging, }) ctx.Response().Header.Set("ETag", etag) diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index b485cd1e..b582860c 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -143,13 +143,19 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au ok := isSpecialPayload(hashPayloadHeader) if !ok { - // Calculate the hash of the request payload - hashedPayload := sha256.Sum256(ctx.Body()) - hexPayload := hex.EncodeToString(hashedPayload[:]) + if utils.IsBigDataAction(ctx) { + rdr := ctx.Request().BodyStream() + r := utils.NewHashReader(rdr, sha256.New(), hashPayloadHeader, utils.HashTypeSha256) + ctx.Locals("body-reader", r) + } else { + // Calculate the hash of the request payload + hashedPayload := sha256.Sum256(ctx.Body()) + hexPayload := hex.EncodeToString(hashedPayload[:]) - // Compare the calculated hash with the hash provided - if hashPayloadHeader != hexPayload { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger}) + // Compare the calculated hash with the hash provided + if hashPayloadHeader != hexPayload { + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger}) + } } } @@ -166,7 +172,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au SecretAccessKey: account.Secret, }, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) { options.DisableURIPathEscaping = true - if debug { + if true { options.LogSigning = true options.Logger = logging.NewStandardLogger(os.Stderr) } @@ -181,6 +187,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au } calculatedSign := strings.Split(parts[3], "=")[1] expectedSign := strings.Split(authParts[2], "=")[1] + fmt.Println(calculatedSign, expectedSign) if expectedSign != calculatedSign { return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger}) diff --git a/s3api/middlewares/md5.go b/s3api/middlewares/md5.go index 2da5f1a9..b1f97209 100644 --- a/s3api/middlewares/md5.go +++ b/s3api/middlewares/md5.go @@ -20,6 +20,7 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/s3api/controllers" + "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3log" ) @@ -31,11 +32,17 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler { return ctx.Next() } - sum := md5.Sum(ctx.Body()) - calculatedSum := base64.StdEncoding.EncodeToString(sum[:]) + if utils.IsBigDataAction(ctx) { + rdr := ctx.Locals("body-reader").(*utils.HashReader) + r := utils.NewHashReader(rdr, md5.New(), incomingSum, utils.HashTypeMd5) + ctx.Locals("body-reader", r) + } else { + sum := md5.Sum(ctx.Body()) + calculatedSum := base64.StdEncoding.EncodeToString(sum[:]) - if incomingSum != calculatedSum { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger}) + if incomingSum != calculatedSum { + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger}) + } } return ctx.Next() diff --git a/s3api/utils/reader.go b/s3api/utils/reader.go new file mode 100644 index 00000000..6dad261d --- /dev/null +++ b/s3api/utils/reader.go @@ -0,0 +1,72 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "encoding/base64" + "encoding/hex" + "errors" + "hash" + "io" + + "github.com/versity/versitygw/s3err" +) + +type HashType string + +const ( + HashTypeMd5 = "md5" + HashTypeSha256 = "sha256" +) + +type HashReader struct { + hashType HashType + hash hash.Hash + r io.Reader + sum string + err error +} + +func NewHashReader(r io.Reader, hash hash.Hash, sum string, ht HashType) *HashReader { + return &HashReader{hash: hash, r: r, sum: sum, hashType: ht} +} + +func (hr *HashReader) Read(p []byte) (int, error) { + n, readerr := hr.r.Read(p) + _, err := hr.hash.Write(p[:n]) + if err != nil { + return n, err + } + if errors.Is(readerr, io.EOF) { + if hr.hashType == HashTypeMd5 { + sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil)) + if sum != hr.sum { + hr.err = s3err.GetAPIError(s3err.ErrInvalidDigest) + return n, s3err.GetAPIError(s3err.ErrInvalidDigest) + } + } else if hr.hashType == HashTypeSha256 { + sum := hex.EncodeToString(hr.hash.Sum(nil)) + if sum != hr.sum { + hr.err = s3err.GetAPIError(s3err.ErrContentSHA256Mismatch) + return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch) + } + } + } + return n, readerr +} + +func (hr *HashReader) Err() error { + return hr.err +} diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 7a0e9cba..9358cf95 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -18,6 +18,7 @@ import ( "bytes" "errors" "fmt" + "io" "net/http" "regexp" "strconv" @@ -51,8 +52,15 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) { req := ctx.Request() + var body io.Reader + if IsBigDataAction(ctx) { + body = req.BodyStream() + fmt.Println("create ctx body: ", body) + } else { + body = bytes.NewReader(req.Body()) + } - httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), bytes.NewReader(req.Body())) + httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body) if err != nil { return nil, errors.New("error in creating an http request") } @@ -131,3 +139,12 @@ func includeHeader(hdr string, signedHdrs []string) bool { } return false } + +func IsBigDataAction(ctx *fiber.Ctx) bool { + if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 { + if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") { + return true + } + } + return false +}