mirror of
https://github.com/versity/versitygw.git
synced 2026-01-08 12:41:10 +00:00
Merge pull request #1272 from versity/sis/debug-logging-chunk-readers
feat: adds debug logging for chunk readers.
This commit is contained in:
@@ -24,11 +24,15 @@ import (
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
type Color string
|
||||
|
||||
const (
|
||||
green Color = "\033[32m"
|
||||
yellow Color = "\033[33m"
|
||||
blue Color = "\033[34m"
|
||||
Purple Color = "\033[0;35m"
|
||||
|
||||
reset = "\033[0m"
|
||||
green = "\033[32m"
|
||||
yellow = "\033[33m"
|
||||
blue = "\033[34m"
|
||||
borderChar = "─"
|
||||
boxWidth = 120
|
||||
)
|
||||
@@ -75,9 +79,7 @@ func LogFiberResponseDetails(ctx *fiber.Ctx) {
|
||||
if !ok {
|
||||
body := ctx.Response().Body()
|
||||
if len(body) != 0 {
|
||||
printBoxTitleLine(blue, "RESPONSE BODY", boxWidth, false)
|
||||
fmt.Printf("%s%s%s\n", blue, body, reset)
|
||||
printHorizontalBorder(blue, boxWidth, false)
|
||||
PrintInsideHorizontalBorders(blue, "RESPONSE BODY", string(body), boxWidth)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -96,12 +98,32 @@ func Logf(format string, v ...any) {
|
||||
return
|
||||
}
|
||||
debugPrefix := "[DEBUG]: "
|
||||
fmt.Printf(yellow+debugPrefix+format+reset+"\n", v...)
|
||||
fmt.Printf(string(yellow)+debugPrefix+format+reset+"\n", v...)
|
||||
}
|
||||
|
||||
// Infof prints out green info block with [INFO]: prefix
|
||||
func Infof(format string, v ...any) {
|
||||
if !debugEnabled.Load() {
|
||||
return
|
||||
}
|
||||
debugPrefix := "[INFO]: "
|
||||
fmt.Printf(string(green)+debugPrefix+format+reset+"\n", v...)
|
||||
}
|
||||
|
||||
// PrintInsideHorizontalBorders prints the text inside horizontal
|
||||
// border and title in the center of upper border
|
||||
func PrintInsideHorizontalBorders(color Color, title, text string, width int) {
|
||||
if !debugEnabled.Load() {
|
||||
return
|
||||
}
|
||||
printBoxTitleLine(color, title, width, false)
|
||||
fmt.Printf("%s%s%s\n", color, text, reset)
|
||||
printHorizontalBorder(color, width, false)
|
||||
}
|
||||
|
||||
// Prints out box title either with closing characters or not: "┌", "┐"
|
||||
// e.g ┌────────────────[ RESPONSE HEADERS ]────────────────┐
|
||||
func printBoxTitleLine(color, title string, length int, closing bool) {
|
||||
func printBoxTitleLine(color Color, title string, length int, closing bool) {
|
||||
leftCorner, rightCorner := "┌", "┐"
|
||||
|
||||
if !closing {
|
||||
@@ -121,22 +143,22 @@ func printBoxTitleLine(color, title string, length int, closing bool) {
|
||||
strings.Repeat(borderChar, rightLen) +
|
||||
rightCorner
|
||||
|
||||
fmt.Println(color + line + reset)
|
||||
fmt.Println(string(color) + line + reset)
|
||||
}
|
||||
|
||||
// Prints out a horizontal line either with closing characters or not: "└", "┘"
|
||||
func printHorizontalBorder(color string, length int, closing bool) {
|
||||
func printHorizontalBorder(color Color, length int, closing bool) {
|
||||
leftCorner, rightCorner := "└", "┘"
|
||||
if !closing {
|
||||
leftCorner, rightCorner = borderChar, borderChar
|
||||
}
|
||||
|
||||
line := leftCorner + strings.Repeat(borderChar, length-2) + rightCorner + reset
|
||||
fmt.Println(color + line)
|
||||
fmt.Println(string(color) + line)
|
||||
}
|
||||
|
||||
// wrapInBox wraps the output of a function call (fn) inside a styled box with a title.
|
||||
func wrapInBox(color, title string, length int, fn func()) {
|
||||
func wrapInBox(color Color, title string, length int, fn func()) {
|
||||
printBoxTitleLine(color, title, length, true)
|
||||
fn()
|
||||
printHorizontalBorder(color, length, true)
|
||||
@@ -154,7 +176,7 @@ func getLen(str string) int {
|
||||
|
||||
// prints a formatted key-value pair within a box layout,
|
||||
// wrapping the value text if it exceeds the allowed width.
|
||||
func printWrappedLine(keyColor, key, value string) {
|
||||
func printWrappedLine(keyColor Color, key, value string) {
|
||||
prefix := fmt.Sprintf("%s│%s %s%-13s%s : ", green, reset, keyColor, key, reset)
|
||||
prefixLen := len(prefix) - len(green) - len(reset) - len(keyColor) - len(reset)
|
||||
// the actual prefix size without colors
|
||||
|
||||
@@ -131,7 +131,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
var err error
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
var cr io.Reader
|
||||
cr, err = utils.NewChunkReader(ctx, r, authData, region, account.Secret, tdate, debug)
|
||||
cr, err = utils.NewChunkReader(ctx, r, authData, region, account.Secret, tdate)
|
||||
return cr
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3api/debuglogger"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
@@ -100,43 +101,49 @@ func IsStreamingPayload(str string) bool {
|
||||
pt == payloadTypeStreamingSignedTrailer
|
||||
}
|
||||
|
||||
func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time, debug bool) (io.Reader, error) {
|
||||
func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secret string, date time.Time) (io.Reader, error) {
|
||||
decContLengthStr := ctx.Get("X-Amz-Decoded-Content-Length")
|
||||
if decContLengthStr == "" {
|
||||
debuglogger.Logf("missing required header 'X-Amz-Decoded-Content-Length'")
|
||||
return nil, s3err.GetAPIError(s3err.ErrMissingContentLength)
|
||||
}
|
||||
decContLength, err := strconv.ParseInt(decContLengthStr, 10, 64)
|
||||
//TODO: not sure if InvalidRequest should be returned in this case
|
||||
if err != nil {
|
||||
debuglogger.Logf("invalid value for 'X-Amz-Decoded-Content-Length': %v", decContLengthStr)
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
if decContLength > maxObjSizeLimit {
|
||||
debuglogger.Logf("the object size exceeds the allowed limit: (size): %v, (limit): %v", decContLength, maxObjSizeLimit)
|
||||
return nil, s3err.GetAPIError(s3err.ErrEntityTooLarge)
|
||||
}
|
||||
|
||||
contentSha256 := payloadType(ctx.Get("X-Amz-Content-Sha256"))
|
||||
if !contentSha256.isValid() {
|
||||
//TODO: Add proper APIError
|
||||
debuglogger.Logf("invalid value for 'X-Amz-Content-Sha256': %v", contentSha256)
|
||||
return nil, fmt.Errorf("invalid x-amz-content-sha256: %v", string(contentSha256))
|
||||
}
|
||||
|
||||
checksumType := checksumType(strings.ToLower(ctx.Get("X-Amz-Trailer")))
|
||||
if contentSha256 != payloadTypeStreamingSigned && !checksumType.isValid() {
|
||||
debuglogger.Logf("invalid value for 'X-Amz-Trailer': %v", checksumType)
|
||||
return nil, s3err.GetAPIError(s3err.ErrTrailerHeaderNotSupported)
|
||||
}
|
||||
|
||||
switch contentSha256 {
|
||||
case payloadTypeStreamingUnsignedTrailer:
|
||||
return NewUnsignedChunkReader(r, checksumType, debug)
|
||||
return NewUnsignedChunkReader(r, checksumType)
|
||||
case payloadTypeStreamingSignedTrailer:
|
||||
return NewSignedChunkReader(r, authdata, region, secret, date, checksumType, debug)
|
||||
return NewSignedChunkReader(r, authdata, region, secret, date, checksumType)
|
||||
case payloadTypeStreamingSigned:
|
||||
return NewSignedChunkReader(r, authdata, region, secret, date, "", debug)
|
||||
return NewSignedChunkReader(r, authdata, region, secret, date, "")
|
||||
// return not supported for:
|
||||
// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
|
||||
// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
|
||||
default:
|
||||
debuglogger.Logf("unsupported chunk reader algorithm: %v", contentSha256)
|
||||
return nil, getPayloadTypeNotSupportedErr(contentSha256)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3api/debuglogger"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
@@ -64,17 +65,15 @@ type ChunkReader struct {
|
||||
checksumHash hash.Hash
|
||||
isEOF bool
|
||||
isFirstHeader bool
|
||||
//TODO: Add debug logging for the reader
|
||||
debug bool
|
||||
region string
|
||||
date time.Time
|
||||
region string
|
||||
date time.Time
|
||||
}
|
||||
|
||||
// NewChunkReader reads from request body io.Reader and parses out the
|
||||
// chunk metadata in stream. The headers are validated for proper signatures.
|
||||
// Reading from the chunk reader will read only the object data stream
|
||||
// without the chunk headers/trailers.
|
||||
func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string, date time.Time, chType checksumType, debug bool) (io.Reader, error) {
|
||||
func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string, date time.Time, chType checksumType) (io.Reader, error) {
|
||||
chRdr := &ChunkReader{
|
||||
r: r,
|
||||
signingKey: getSigningKey(secret, region, date),
|
||||
@@ -86,17 +85,22 @@ func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string,
|
||||
date: date,
|
||||
region: region,
|
||||
trailer: chType,
|
||||
debug: debug,
|
||||
}
|
||||
|
||||
if chType != "" {
|
||||
checksumHasher, err := getHasher(chType)
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to initialize hash calculator: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
chRdr.checksumHash = checksumHasher
|
||||
}
|
||||
if chType == "" {
|
||||
debuglogger.Infof("initializing signed chunk reader")
|
||||
} else {
|
||||
debuglogger.Infof("initializing signed chunk reader with '%v' trailing checksum", chType)
|
||||
}
|
||||
return chRdr, nil
|
||||
}
|
||||
|
||||
@@ -153,11 +157,13 @@ func (cr *ChunkReader) getStringToSignPrefix(algo string) string {
|
||||
func (cr *ChunkReader) getChunkStringToSign() string {
|
||||
prefix := cr.getStringToSignPrefix(streamPayloadAlgo)
|
||||
chunkHash := cr.chunkHash.Sum(nil)
|
||||
return fmt.Sprintf("%s\n%s\n%s\n%s",
|
||||
strToSign := fmt.Sprintf("%s\n%s\n%s\n%s",
|
||||
prefix,
|
||||
cr.prevSig,
|
||||
zeroLenSig,
|
||||
hex.EncodeToString(chunkHash))
|
||||
debuglogger.PrintInsideHorizontalBorders(debuglogger.Purple, "STRING TO SIGN", strToSign, 64)
|
||||
return strToSign
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html#example-signature-calculations-trailing-header
|
||||
@@ -169,11 +175,15 @@ func (cr *ChunkReader) getTrailerChunkStringToSign() string {
|
||||
|
||||
prefix := cr.getStringToSignPrefix(streamPayloadTrailerAlgo)
|
||||
|
||||
return fmt.Sprintf("%s\n%s\n%s",
|
||||
strToSign := fmt.Sprintf("%s\n%s\n%s",
|
||||
prefix,
|
||||
cr.prevSig,
|
||||
sig,
|
||||
)
|
||||
|
||||
debuglogger.PrintInsideHorizontalBorders(debuglogger.Purple, "TRAILER STRING TO SIGN", strToSign, 64)
|
||||
|
||||
return strToSign
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html#example-signature-calculations-trailing-header
|
||||
@@ -183,6 +193,7 @@ func (cr *ChunkReader) verifyTrailerSignature() error {
|
||||
sig := hex.EncodeToString(hmac256(cr.signingKey, []byte(strToSign)))
|
||||
|
||||
if sig != cr.trailerSig {
|
||||
debuglogger.Logf("incorrect trailing signature: (calculated): %v, (got): %v", sig, cr.trailerSig)
|
||||
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
|
||||
@@ -195,6 +206,7 @@ func (cr *ChunkReader) verifyChecksum() error {
|
||||
checksum := base64.StdEncoding.EncodeToString(checksumHash)
|
||||
if checksum != cr.parsedChecksum {
|
||||
algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(string(cr.trailer), "x-amz-checksum-")))
|
||||
debuglogger.Logf("incorrect trailing checksum: (calculated): %v, (got): %v", checksum, cr.parsedChecksum)
|
||||
return s3err.GetChecksumBadDigestErr(algo)
|
||||
}
|
||||
|
||||
@@ -208,6 +220,7 @@ func (cr *ChunkReader) checkSignature() error {
|
||||
cr.prevSig = hex.EncodeToString(hmac256(cr.signingKey, []byte(sigstr)))
|
||||
|
||||
if cr.prevSig != cr.parsedSig {
|
||||
debuglogger.Logf("incorrect signature: (calculated): %v, (got) %v", cr.prevSig, cr.parsedSig)
|
||||
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
cr.parsedSig = ""
|
||||
@@ -239,12 +252,14 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to parse chunk headers: %v", err)
|
||||
return 0, err
|
||||
}
|
||||
cr.parsedSig = sig
|
||||
// If we hit the final chunk, calculate and validate the final
|
||||
// chunk signature and finish reading
|
||||
if chunkSize == 0 {
|
||||
debuglogger.Infof("final chunk parsed:\nchunk size: %v\nsignature: %v\nbuffer offset: %v", chunkSize, sig, bufOffset)
|
||||
cr.chunkHash.Reset()
|
||||
err := cr.checkSignature()
|
||||
if err != nil {
|
||||
@@ -252,6 +267,7 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
|
||||
}
|
||||
|
||||
if cr.trailer != "" {
|
||||
debuglogger.Infof("final chunk trailers parsed:\nchecksum: %v\ntrailing signature: %v", cr.parsedChecksum, cr.trailerSig)
|
||||
err := cr.verifyChecksum()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -264,6 +280,7 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
|
||||
|
||||
return 0, io.EOF
|
||||
}
|
||||
debuglogger.Infof("chunk headers parsed:\nchunk size: %v\nsignature: %v\nbuffer offset: %v", chunkSize, sig, bufOffset)
|
||||
|
||||
// move data up to remove chunk header
|
||||
copy(p, p[bufOffset:n])
|
||||
@@ -279,6 +296,7 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
|
||||
}
|
||||
n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
|
||||
if (chunkSize + int64(n)) > math.MaxInt {
|
||||
debuglogger.Logf("exceeding the limit of maximum integer allowed: (value): %v, (limit): %v", chunkSize+int64(n), math.MaxInt)
|
||||
return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
return n + int(chunkSize), err
|
||||
@@ -301,6 +319,7 @@ func getSigningKey(secret, region string, date time.Time) []byte {
|
||||
dateRegionKey := hmac256(dateKey, []byte(region))
|
||||
dateRegionServiceKey := hmac256(dateRegionKey, []byte(awsS3Service))
|
||||
signingKey := hmac256(dateRegionServiceKey, []byte(awsV4Request))
|
||||
debuglogger.Infof("signing key: %s", hex.EncodeToString(signingKey))
|
||||
return signingKey
|
||||
}
|
||||
|
||||
@@ -325,9 +344,11 @@ const (
|
||||
func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
|
||||
stashLen := len(cr.stash)
|
||||
if stashLen > maxHeaderSize {
|
||||
debuglogger.Logf("the stash length exceeds the maximum allowed chunk header size: (stash len): %v, (header limit): %v", stashLen, maxHeaderSize)
|
||||
return 0, "", 0, errInvalidChunkFormat
|
||||
}
|
||||
if cr.stash != nil {
|
||||
debuglogger.Logf("recovering the stash: (stash len): %v", stashLen)
|
||||
tmp := make([]byte, stashLen+len(header))
|
||||
copy(tmp, cr.stash)
|
||||
copy(tmp[len(cr.stash):], header)
|
||||
@@ -342,6 +363,7 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
|
||||
if !cr.isFirstHeader {
|
||||
err := readAndSkip(rdr, '\r', '\n')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read chunk header first 2 bytes: (should be): \\r\\n, (got): %q", header[:2])
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
}
|
||||
@@ -349,20 +371,24 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
|
||||
// read and parse the chunk size
|
||||
chunkSizeStr, err := readAndTrim(rdr, ';')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read chunk size: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
chunkSize, err := strconv.ParseInt(chunkSizeStr, 16, 64)
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to parse chunk size: (size): %v, (err): %v", chunkSizeStr, err)
|
||||
return 0, "", 0, errInvalidChunkFormat
|
||||
}
|
||||
|
||||
// read the chunk signature
|
||||
err = readAndSkip(rdr, 'c', 'h', 'u', 'n', 'k', '-', 's', 'i', 'g', 'n', 'a', 't', 'u', 'r', 'e', '=')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read 'chunk-signature=': %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
sig, err := readAndTrim(rdr, '\r')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read '\\r', after chunk signature: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
|
||||
@@ -371,14 +397,17 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
|
||||
if cr.trailer != "" {
|
||||
err = readAndSkip(rdr, '\n')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read \\n before the trailer: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
// parse and validate the trailing header
|
||||
trailer, err := readAndTrim(rdr, ':')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read trailer prefix: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
if trailer != string(cr.trailer) {
|
||||
debuglogger.Logf("incorrect trailer prefix: (expected): %v, (got): %v", cr.trailer, trailer)
|
||||
return 0, "", 0, errInvalidChunkFormat
|
||||
}
|
||||
|
||||
@@ -387,30 +416,36 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
|
||||
// parse the checksum
|
||||
checksum, err := readAndTrim(rdr, '\r')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read checksum value: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
|
||||
if !IsValidChecksum(checksum, algo) {
|
||||
debuglogger.Logf("invalid checksum value: %v", checksum)
|
||||
return 0, "", 0, s3err.GetInvalidTrailingChecksumHeaderErr(trailer)
|
||||
}
|
||||
|
||||
err = readAndSkip(rdr, '\n')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read \\n after checksum: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
|
||||
// parse the trailing signature
|
||||
trailerSigPrefix, err := readAndTrim(rdr, ':')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read trailing signature prefix: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
|
||||
if trailerSigPrefix != trailerSignatureHeader {
|
||||
debuglogger.Logf("invalid trailing signature prefix: (expected): %v, (got): %v", trailerSignatureHeader, trailerSigPrefix)
|
||||
return 0, "", 0, errInvalidChunkFormat
|
||||
}
|
||||
|
||||
trailerSig, err := readAndTrim(rdr, '\r')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read trailing signature: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
|
||||
@@ -421,6 +456,7 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
|
||||
// "\r\n\r\n" is followed after the last chunk
|
||||
err = readAndSkip(rdr, '\n', '\r', '\n')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read \\n\\r\\n at the end of chunk header: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
|
||||
@@ -429,6 +465,7 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
|
||||
|
||||
err = readAndSkip(rdr, '\n')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read \\n at the end of chunk header: %v", err)
|
||||
return cr.handleRdrErr(err, header)
|
||||
}
|
||||
|
||||
@@ -451,6 +488,7 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
|
||||
func (cr *ChunkReader) stashAndSkipHeader(header []byte) (int64, string, int, error) {
|
||||
cr.stash = make([]byte, len(header))
|
||||
copy(cr.stash, header)
|
||||
debuglogger.Logf("stashing the header: (header length): %v", len(header))
|
||||
return 0, "", 0, errskipHeader
|
||||
}
|
||||
|
||||
@@ -460,6 +498,7 @@ func (cr *ChunkReader) stashAndSkipHeader(header []byte) (int64, string, int, er
|
||||
func (cr *ChunkReader) handleRdrErr(err error, header []byte) (int64, string, int, error) {
|
||||
if err == io.EOF {
|
||||
if cr.isEOF {
|
||||
debuglogger.Logf("incomplete chunk encoding, EOF reached")
|
||||
return 0, "", 0, errInvalidChunkFormat
|
||||
}
|
||||
return cr.stashAndSkipHeader(header)
|
||||
|
||||
@@ -29,6 +29,8 @@ import (
|
||||
"math/bits"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/versity/versitygw/s3api/debuglogger"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -42,30 +44,28 @@ type UnsignedChunkReader struct {
|
||||
expectedChecksum string
|
||||
hasher hash.Hash
|
||||
stash []byte
|
||||
chunkCounter int
|
||||
offset int
|
||||
//TODO: Add debug logging for the reader
|
||||
debug bool
|
||||
}
|
||||
|
||||
func NewUnsignedChunkReader(r io.Reader, ct checksumType, debug bool) (*UnsignedChunkReader, error) {
|
||||
func NewUnsignedChunkReader(r io.Reader, ct checksumType) (*UnsignedChunkReader, error) {
|
||||
hasher, err := getHasher(ct)
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to initialize hash calculator: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
debuglogger.Infof("initializing unsigned chunk reader")
|
||||
return &UnsignedChunkReader{
|
||||
reader: bufio.NewReader(r),
|
||||
checksumType: ct,
|
||||
stash: make([]byte, 0),
|
||||
hasher: hasher,
|
||||
chunkCounter: 1,
|
||||
debug: debug,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ucr *UnsignedChunkReader) Read(p []byte) (int, error) {
|
||||
// First read any stashed data
|
||||
if len(ucr.stash) != 0 {
|
||||
debuglogger.Infof("recovering the stash: (stash length): %v", len(ucr.stash))
|
||||
n := copy(p, ucr.stash)
|
||||
ucr.offset += n
|
||||
|
||||
@@ -92,22 +92,24 @@ func (ucr *UnsignedChunkReader) Read(p []byte) (int, error) {
|
||||
// Read and cache the payload
|
||||
_, err = io.ReadFull(rdr, payload)
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read chunk data: %v", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Skip the trailing "\r\n"
|
||||
if err := ucr.readAndSkip('\r', '\n'); err != nil {
|
||||
debuglogger.Logf("failed to read trailing \\r\\n after chunk data: %v", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Copy the payload into the io.Reader buffer
|
||||
n := copy(p[ucr.offset:], payload)
|
||||
ucr.offset += n
|
||||
ucr.chunkCounter++
|
||||
|
||||
if int64(n) < chunkSize {
|
||||
// stash the remaining data
|
||||
ucr.stash = payload[n:]
|
||||
debuglogger.Infof("stashing the remaining data: (stash length): %v", len(ucr.stash))
|
||||
dataRead := ucr.offset
|
||||
ucr.offset = 0
|
||||
return dataRead, nil
|
||||
@@ -116,6 +118,7 @@ func (ucr *UnsignedChunkReader) Read(p []byte) (int, error) {
|
||||
|
||||
// Read and validate trailers
|
||||
if err := ucr.readTrailer(); err != nil {
|
||||
debuglogger.Logf("failed to read trailer: %v", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@@ -145,15 +148,19 @@ func (ucr *UnsignedChunkReader) readAndSkip(data ...byte) error {
|
||||
func (ucr *UnsignedChunkReader) extractChunkSize() (int64, error) {
|
||||
line, err := ucr.reader.ReadString('\n')
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to parse chunk size: %v", err)
|
||||
return 0, errMalformedEncoding
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
|
||||
chunkSize, err := strconv.ParseInt(line, 16, 64)
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to convert chunk size: %v", err)
|
||||
return 0, errMalformedEncoding
|
||||
}
|
||||
|
||||
debuglogger.Infof("chunk size extracted: %v", chunkSize)
|
||||
|
||||
return chunkSize, nil
|
||||
}
|
||||
|
||||
@@ -164,6 +171,7 @@ func (ucr *UnsignedChunkReader) readTrailer() error {
|
||||
for {
|
||||
v, err := ucr.reader.ReadByte()
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read byte: %v", err)
|
||||
if err == io.EOF {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
@@ -176,12 +184,14 @@ func (ucr *UnsignedChunkReader) readTrailer() error {
|
||||
var tmp [3]byte
|
||||
_, err = io.ReadFull(ucr.reader, tmp[:])
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to read chunk ending: \\n\\r\\n: %v", err)
|
||||
if err == io.EOF {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(tmp[:], trailerDelim) {
|
||||
debuglogger.Logf("incorrect trailer delimiter: (expected): \\n\\r\\n, (got): %q", tmp[:])
|
||||
return errMalformedEncoding
|
||||
}
|
||||
break
|
||||
@@ -192,15 +202,18 @@ func (ucr *UnsignedChunkReader) readTrailer() error {
|
||||
trailerHeader = strings.TrimSpace(trailerHeader)
|
||||
trailerHeaderParts := strings.Split(trailerHeader, ":")
|
||||
if len(trailerHeaderParts) != 2 {
|
||||
debuglogger.Logf("invalid trailer header parts: %v", trailerHeaderParts)
|
||||
return errMalformedEncoding
|
||||
}
|
||||
|
||||
if trailerHeaderParts[0] != string(ucr.checksumType) {
|
||||
debuglogger.Logf("invalid checksum type: %v", trailerHeaderParts[0])
|
||||
//TODO: handle the error
|
||||
return errMalformedEncoding
|
||||
}
|
||||
|
||||
ucr.expectedChecksum = trailerHeaderParts[1]
|
||||
debuglogger.Infof("parsed the trailing header:\n%v:%v", trailerHeaderParts[0], trailerHeaderParts[1])
|
||||
|
||||
// Validate checksum
|
||||
return ucr.validateChecksum()
|
||||
@@ -212,6 +225,7 @@ func (ucr *UnsignedChunkReader) validateChecksum() error {
|
||||
checksum := base64.StdEncoding.EncodeToString(csum)
|
||||
|
||||
if checksum != ucr.expectedChecksum {
|
||||
debuglogger.Logf("incorrect checksum: (expected): %v, (got): %v", ucr.expectedChecksum, checksum)
|
||||
return fmt.Errorf("actual checksum: %v, expected checksum: %v", checksum, ucr.expectedChecksum)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user