mirror of
https://github.com/versity/versitygw.git
synced 2026-01-09 13:03:09 +00:00
d861dc8e30
Fixes #1600 Fixes #1603 Fixes #1607 Fixes #1626 Fixes #1632 Fixes #1652 Fixes #1653 Fixes #1656 Fixes #1657 Fixes #1659 This PR focuses mainly on unsigned streaming payload **trailer request payload parsing** and **checksum calculation**. For streaming uploads, there are essentially two ways to specify checksums: 1. via `x-amz-checksum-*` headers, 2. via `x-amz-trailer`, or none — in which case the checksum should default to **crc64nvme**. Previously, the implementation calculated the checksum only from `x-amz-checksum-*` headers. Now, `x-amz-trailer` is also treated as a checksum-related header and indicates the checksum algorithm for streaming requests. If `x-amz-trailer` is present, the payload must include a trailing checksum; otherwise, an error is returned. `x-amz-trailer` and any `x-amz-checksum-*` header **cannot** be used together — doing so results in an error. If `x-amz-sdk-checksum-algorithm` is specified, then either `x-amz-trailer` or one of the `x-amz-checksum-*` headers must also be present, and the algorithms must match. If they don’t, an error is returned. The old implementation used to return an internal error when no `x-amz-trailer` was received in streaming requests or when the payload didn’t contain a trailer. This is now fixed. Checksum calculation used to happen twice in the gateway (once in the chunk reader and once in the backend). A new `ChecksumReader` is introduced to prevent double computation, and the trailing checksum is now read by the backend from the chunk reader. The logic for stacking `io.Reader`s in the Fiber context is preserved, but extended: once a `ChecksumReader` is stacked, all following `io.Reader`s are wrapped with `MockChecksumReader`, which simply delegates to the underlying checksum reader. In the backend, a simple type assertion on `io.Reader` provides the necessary checksum metadata (algorithm, value, etc.).
105 lines
3.1 KiB
Go
105 lines
3.1 KiB
Go
// Copyright 2023 Versity Software
|
|
// This file is licensed under the Apache License, Version 2.0
|
|
// (the "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
package middlewares
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
|
|
"github.com/gofiber/fiber/v2"
|
|
"github.com/versity/versitygw/s3api/utils"
|
|
)
|
|
|
|
// ChecksumReader extends io.Reader with checksum-related metadata.
|
|
// It is used to differentiate normal readers from readers that can
|
|
// report a checksum and the algorithm used to produce it.
|
|
type ChecksumReader interface {
|
|
io.Reader
|
|
Algorithm() string
|
|
Checksum() string
|
|
}
|
|
|
|
// NewChecksumReader wraps a stackedReader and returns a reader that
|
|
// preserves checksum behavior when the *original* bodyReader implemented
|
|
// ChecksumReader.
|
|
//
|
|
// If bodyReader already supports ChecksumReader, we wrap stackedReader
|
|
// with MockChecksumReader so that reading continues from stackedReader,
|
|
// but Algorithm() and Checksum() still delegate to the underlying reader.
|
|
//
|
|
// If bodyReader is not a ChecksumReader, we simply return stackedReader.
|
|
func NewChecksumReader(bodyReader io.Reader, stackedReader io.Reader) io.Reader {
|
|
_, ok := bodyReader.(ChecksumReader)
|
|
if ok {
|
|
return &MockChecksumReader{rdr: stackedReader}
|
|
}
|
|
|
|
return stackedReader
|
|
}
|
|
|
|
// MockChecksumReader is a wrapper around an io.Reader that forwards Read()
|
|
// but also conditionally exposes checksum metadata if the underlying reader
|
|
// implements the ChecksumReader interface.
|
|
type MockChecksumReader struct {
|
|
rdr io.Reader
|
|
}
|
|
|
|
// Read simply forwards data reads to the underlying reader.
|
|
func (rr *MockChecksumReader) Read(buffer []byte) (int, error) {
|
|
return rr.rdr.Read(buffer)
|
|
}
|
|
|
|
// Algorithm returns the checksum algorithm used by the underlying reader,
|
|
// but only if the wrapped reader implements ChecksumReader.
|
|
func (rr *MockChecksumReader) Algorithm() string {
|
|
r, ok := rr.rdr.(ChecksumReader)
|
|
if ok {
|
|
return r.Algorithm()
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// Checksum returns the checksum value from the underlying reader,
|
|
// if it implements ChecksumReader. Otherwise returns an empty string.
|
|
func (rr *MockChecksumReader) Checksum() string {
|
|
r, ok := rr.rdr.(ChecksumReader)
|
|
if ok {
|
|
return r.Checksum()
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
var _ ChecksumReader = &MockChecksumReader{}
|
|
|
|
func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
|
|
rdr, ok := utils.ContextKeyBodyReader.Get(ctx).(io.Reader)
|
|
if !ok {
|
|
rdr = ctx.Request().BodyStream()
|
|
// Override the body reader with an empty reader to prevent panics
|
|
// in case of unexpected or malformed HTTP requests.
|
|
if rdr == nil {
|
|
rdr = bytes.NewBuffer([]byte{})
|
|
}
|
|
}
|
|
|
|
r := wr(rdr)
|
|
// Ensure checksum behavior is stacked if the original body reader had it.
|
|
r = NewChecksumReader(rdr, r)
|
|
|
|
utils.ContextKeyBodyReader.Set(ctx, r)
|
|
}
|