feat: Closes #290, implemented request body stream reading for PutObject and UploadPart actions.

s3api/utils/reader.go

@@ -0,0 +1,72 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"hash"
+	"io"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+type HashType string
+
+const (
+	HashTypeMd5    = "md5"
+	HashTypeSha256 = "sha256"
+)
+
+type HashReader struct {
+	hashType HashType
+	hash     hash.Hash
+	r        io.Reader
+	sum      string
+	err      error
+}
+
+func NewHashReader(r io.Reader, hash hash.Hash, sum string, ht HashType) *HashReader {
+	return &HashReader{hash: hash, r: r, sum: sum, hashType: ht}
+}
+
+func (hr *HashReader) Read(p []byte) (int, error) {
+	n, readerr := hr.r.Read(p)
+	_, err := hr.hash.Write(p[:n])
+	if err != nil {
+		return n, err
+	}
+	if errors.Is(readerr, io.EOF) {
+		if hr.hashType == HashTypeMd5 {
+			sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
+			if sum != hr.sum {
+				hr.err = s3err.GetAPIError(s3err.ErrInvalidDigest)
+				return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
+			}
+		} else if hr.hashType == HashTypeSha256 {
+			sum := hex.EncodeToString(hr.hash.Sum(nil))
+			if sum != hr.sum {
+				hr.err = s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
+				return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
+			}
+		}
+	}
+	return n, readerr
+}
+
+func (hr *HashReader) Err() error {
+	return hr.err
+}

s3api/utils/utils.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -51,8 +52,15 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin
 
 func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
 	req := ctx.Request()
+	var body io.Reader
+	if IsBigDataAction(ctx) {
+		body = req.BodyStream()
+		fmt.Println("create ctx body: ", body)
+	} else {
+		body = bytes.NewReader(req.Body())
+	}
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), bytes.NewReader(req.Body()))
+	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -131,3 +139,12 @@ func includeHeader(hdr string, signedHdrs []string) bool {
 	}
 	return false
 }
+
+func IsBigDataAction(ctx *fiber.Ctx) bool {
+	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
+		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
+			return true
+		}
+	}
+	return false
+}