mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-21 06:36:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: migrate Fiber to v3.3.0

Browse Source 
Fixes #2180
Fixes #2181

Migrate the gateway from Fiber v2 to Fiber v3.3.0 and update the affected server, middleware, handler, controller, and test code for the new APIs.

Replace the deprecated Fiber filesystem middleware used by the WebUI with the Fiber v3 static middleware, serving the embedded WebUI assets from an fs.Sub filesystem.

Fix the request header limit handling regression by adding a temporary handler for Fiber v3/fasthttp small-buffer errors so oversized request headers return the expected regulated S3 error response.

Fix the debuglogger panic by reworking the boxed key/value formatter used for debug request and response dumps. The formatter now handles long header keys and values without producing invalid wrap widths, negative padding, or out-of-range string slices.
This commit is contained in:
niksis02
2026-06-15 14:48:31 +04:00
parent 619fdb8dce
commit 4d391cabc8
83 changed files with 855 additions and 644 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3api/utils/auth-reader.go
+2 -2
View File
@@ -26,7 +26,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/smithy-go/logging"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
v4 "github.com/versity/versitygw/aws/signer/v4"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
@@ -53,7 +53,7 @@ const (
)
// CheckValidSignature validates the ctx v4 auth signature
func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64) (string, error) {
func CheckValidSignature(ctx fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64) (string, error) {
signedHdrs := strings.Split(auth.SignedHeaders, ";")
// Create a new http request instance from fasthttp request
s3api/utils/auth_test.go
+18 -9
View File
@@ -20,7 +20,8 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
"github.com/valyala/fasthttp"
"github.com/valyala/fasthttp/fasthttputil"
v4 "github.com/versity/versitygw/aws/signer/v4"
)
@@ -84,14 +85,14 @@ func Test_Client_UserAgent(t *testing.T) {
expectedSig := "37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6"
dateStr := "20240206T210328Z"
app := fiber.New(fiber.Config{DisableStartupMessage: true})
app := fiber.New(fiber.Config{})
tdate, err := time.Parse(iso8601Format, dateStr)
if err != nil {
t.Fatal(err)
}
app.Get("/", func(c *fiber.Ctx) error {
app.Get("/", func(c fiber.Ctx) error {
req, err := createHttpRequestFromCtx(c, signedHdrs, int64(c.Request().Header.ContentLength()))
if err != nil {
t.Fatal(err)
@@ -135,10 +136,18 @@ func Test_Client_UserAgent(t *testing.T) {
}
}()
c := fiber.AcquireClient()
c.UserAgent = agent
a := c.Get("http://example.com")
a.HostClient.Dial = func(_ string) (net.Conn, error) { return ln.Dial() }
a.String()
fiber.ReleaseClient(c)
client := fasthttp.Client{
Dial: func(_ string) (net.Conn, error) { return ln.Dial() },
}
req := fasthttp.AcquireRequest()
resp := fasthttp.AcquireResponse()
defer fasthttp.ReleaseRequest(req)
defer fasthttp.ReleaseResponse(resp)
req.SetRequestURI("http://example.com")
req.Header.SetUserAgent(agent)
if err := client.Do(req, resp); err != nil {
t.Fatal(err)
}
}
s3api/utils/chunk-reader.go
+4 -4
View File
@@ -23,7 +23,7 @@ import (
"strings"
"time"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
)
@@ -100,7 +100,7 @@ func (c checksumType) isValid() bool {
}
// Extracts and validates the checksum type from the 'X-Amz-Trailer' header
func ExtractChecksumType(ctx *fiber.Ctx) (checksumType, error) {
func ExtractChecksumType(ctx fiber.Ctx) (checksumType, error) {
trailer := ctx.Get("X-Amz-Trailer")
chType := checksumType(strings.ToLower(trailer))
if chType != "" && !chType.isValid() {
@@ -172,7 +172,7 @@ func IsStreamingPayload(str string) bool {
// ParseDecodedContentLength extracts and validates the
// 'x-amz-decoded-content-length' from fiber context
func ParseDecodedContentLength(ctx *fiber.Ctx) (int64, error) {
func ParseDecodedContentLength(ctx fiber.Ctx) (int64, error) {
decContLengthStr := ctx.Get("X-Amz-Decoded-Content-Length")
if decContLengthStr == "" {
debuglogger.Logf("missing required header 'X-Amz-Decoded-Content-Length'")
@@ -192,7 +192,7 @@ func ParseDecodedContentLength(ctx *fiber.Ctx) (int64, error) {
return decContLength, nil
}
func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, canonicalString, secret string, date time.Time) (io.Reader, error) {
func NewChunkReader(ctx fiber.Ctx, r io.Reader, authdata AuthData, canonicalString, secret string, date time.Time) (io.Reader, error) {
cLength, err := ParseDecodedContentLength(ctx)
if err != nil {
return nil, err
s3api/utils/context-keys.go
+5 -5
View File
@@ -15,7 +15,7 @@
package utils
import (
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
)
// Region, StartTime, IsRoot, Account, AccessKey context locals
@@ -43,19 +43,19 @@ const (
ContextKeyWebsiteConfig ContextKey = "website-config"
)
func (ck ContextKey) Set(ctx *fiber.Ctx, val any) {
func (ck ContextKey) Set(ctx fiber.Ctx, val any) {
ctx.Locals(string(ck), val)
}
func (ck ContextKey) IsSet(ctx *fiber.Ctx) bool {
func (ck ContextKey) IsSet(ctx fiber.Ctx) bool {
val := ctx.Locals(string(ck))
return val != nil
}
func (ck ContextKey) Delete(ctx *fiber.Ctx) {
func (ck ContextKey) Delete(ctx fiber.Ctx) {
ctx.Locals(string(ck), nil)
}
func (ck ContextKey) Get(ctx *fiber.Ctx) any {
func (ck ContextKey) Get(ctx fiber.Ctx) any {
return ctx.Locals(string(ck))
}
s3api/utils/precondition.go
+5 -5
View File
@@ -18,7 +18,7 @@ import (
"strconv"
"time"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
"github.com/versity/versitygw/debuglogger"
)
@@ -45,7 +45,7 @@ func WithCopySource() preconditionOpt {
// - If-None-Match
// - If-Modified-Since
// - If-Unmodified-Since
func ParsePreconditionHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) ConditionalHeaders {
func ParsePreconditionHeaders(ctx fiber.Ctx, opts ...preconditionOpt) ConditionalHeaders {
ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx, opts...)
ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx, opts...)
@@ -58,7 +58,7 @@ func ParsePreconditionHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) Condition
}
// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx
func ParsePreconditionMatchHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*string, *string) {
func ParsePreconditionMatchHeaders(ctx fiber.Ctx, opts ...preconditionOpt) (*string, *string) {
cfg := new(precondtionCfg)
for _, opt := range opts {
opt(cfg)
@@ -75,7 +75,7 @@ func ParsePreconditionMatchHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*st
// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since"
// headers from fiber context to *time.Time
func ParsePreconditionDateHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*time.Time, *time.Time) {
func ParsePreconditionDateHeaders(ctx fiber.Ctx, opts ...preconditionOpt) (*time.Time, *time.Time) {
cfg := new(precondtionCfg)
for _, opt := range opts {
opt(cfg)
@@ -129,7 +129,7 @@ func ParsePreconditionDateHeader(date string) *time.Time {
// ParseIfMatchSize parses the 'x-amz-if-match-size' to *int64
// if parsing fails, returns nil
func ParseIfMatchSize(ctx *fiber.Ctx) *int64 {
func ParseIfMatchSize(ctx fiber.Ctx) *int64 {
ifMatchSizeHdr := ctx.Get("x-amz-if-match-size")
if ifMatchSizeHdr == "" {
return nil
s3api/utils/presign-auth-reader.go
+6 -6
View File
@@ -24,7 +24,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/smithy-go/logging"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
v4 "github.com/versity/versitygw/aws/signer/v4"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
@@ -38,7 +38,7 @@ const (
)
// CheckPresignedSignature validates presigned request signature
func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string) error {
func CheckPresignedSignature(ctx fiber.Ctx, auth AuthData, secret string) error {
signedHdrs := strings.Split(auth.SignedHeaders, ";")
var contentLength int64
@@ -60,7 +60,7 @@ func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string) error
date, _ := time.Parse(iso8601Format, auth.Date)
signer := v4.NewSigner()
uri, _, signMeta, signErr := signer.PresignHTTP(ctx.Context(), aws.Credentials{
uri, _, signMeta, signErr := signer.PresignHTTP(ctx.RequestCtx(), aws.Credentials{
AccessKeyID: auth.Access,
SecretAccessKey: secret,
}, req, unsignedPayload, service, auth.Region, date, signedHdrs, func(options *v4.SignerOptions) {
@@ -104,7 +104,7 @@ func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string) error
// &X-Amz-Expires=86400
// &X-Amz-SignedHeaders=host
// &X-Amz-Signature=1e68ad45c1db540284a4a1eca3884c293ba1a0ff63ab9db9a15b5b29dfa02cd8
func ParsePresignedURIParts(ctx *fiber.Ctx, region string) (AuthData, error) {
func ParsePresignedURIParts(ctx fiber.Ctx, region string) (AuthData, error) {
a := AuthData{}
// Get and verify algorithm query parameter
@@ -218,7 +218,7 @@ func validateAlgorithm(algo string) error {
// IsPresignedURLAuth determines if the request is presigned:
// which is authorization with query params
func IsPresignedURLAuth(ctx *fiber.Ctx) bool {
func IsPresignedURLAuth(ctx fiber.Ctx) bool {
algo := ctx.Query("X-Amz-Algorithm")
creds := ctx.Query("X-Amz-Credential")
signature := ctx.Query("X-Amz-Signature")
@@ -230,7 +230,7 @@ func IsPresignedURLAuth(ctx *fiber.Ctx) bool {
// IsPresignedURLAuthV2 determines if the request is
// query-string signed with aws v2 signer
func IsPresignedURLAuthV2(ctx *fiber.Ctx) bool {
func IsPresignedURLAuthV2(ctx fiber.Ctx) bool {
expires := ctx.Query("Expires")
access := ctx.Query("AWSAccessKeyId")
signature := ctx.Query("Signature")
s3api/utils/request_ids.go
+4 -4
View File
@@ -18,7 +18,7 @@ import (
"crypto/rand"
"encoding/base64"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
"github.com/versity/versitygw/debuglogger"
)
@@ -70,7 +70,7 @@ func NewS3HostID() string {
// EnsureRequestIDs makes sure the request-local IDs exist and are present
// in the response headers. Existing local values are reused so headers and XML
// bodies stay consistent throughout the request.
func EnsureRequestIDs(ctx *fiber.Ctx) (requestID, hostID string) {
func EnsureRequestIDs(ctx fiber.Ctx) (requestID, hostID string) {
requestID = RequestID(ctx)
if requestID == "" {
requestID = NewS3RequestID()
@@ -89,7 +89,7 @@ func EnsureRequestIDs(ctx *fiber.Ctx) (requestID, hostID string) {
return requestID, hostID
}
func RequestID(ctx *fiber.Ctx) string {
func RequestID(ctx fiber.Ctx) string {
requestID, _ := ContextKeyRequestID.Get(ctx).(string)
if requestID != "" {
return requestID
@@ -98,7 +98,7 @@ func RequestID(ctx *fiber.Ctx) string {
return string(ctx.Response().Header.Peek(HeaderAmzRequestID))
}
func HostID(ctx *fiber.Ctx) string {
func HostID(ctx fiber.Ctx) string {
hostID, _ := ContextKeyHostID.Get(ctx).(string)
if hostID != "" {
return hostID
s3api/utils/request_ids_test.go
+1 -1
View File
@@ -19,7 +19,7 @@ import (
"regexp"
"testing"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
"github.com/stretchr/testify/assert"
"github.com/valyala/fasthttp"
)
s3api/utils/signed_headers_test.go
+3 -3
View File
@@ -21,7 +21,7 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
"github.com/stretchr/testify/require"
"github.com/valyala/fasthttp"
v4 "github.com/versity/versitygw/aws/signer/v4"
@@ -151,7 +151,7 @@ func buildPresignedURL(t *testing.T, headers http.Header) string {
return signedURL
}
func signedHeaderAuthCtx(t *testing.T, signedHeaders, extraHeaders http.Header) (*fiber.Ctx, AuthData, time.Time) {
func signedHeaderAuthCtx(t *testing.T, signedHeaders, extraHeaders http.Header) (fiber.Ctx, AuthData, time.Time) {
t.Helper()
signingTime := time.Now().UTC()
@@ -192,7 +192,7 @@ func signedHeaderAuthCtx(t *testing.T, signedHeaders, extraHeaders http.Header)
return ctx, authData, signingTime
}
func fiberCtxFromURL(t *testing.T, method, rawURL string, headers http.Header) *fiber.Ctx {
func fiberCtxFromURL(t *testing.T, method, rawURL string, headers http.Header) fiber.Ctx {
t.Helper()
parsedURL, err := url.Parse(rawURL)
s3api/utils/utils.go
+19 -19
View File
@@ -32,7 +32,7 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
"github.com/valyala/fasthttp"
signerV4 "github.com/versity/versitygw/aws/signer/v4"
"github.com/versity/versitygw/debuglogger"
@@ -135,7 +135,7 @@ func ExtractMetadataFromFields(fields map[string]string) (map[string]string, err
return metadata, nil
}
func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
func createHttpRequestFromCtx(ctx fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
req := ctx.Request()
uri := ctx.OriginalURL()
@@ -180,7 +180,7 @@ var (
}
)
func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
func createPresignedHttpRequestFromCtx(ctx fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
req := ctx.Request()
uri, _, _ := strings.Cut(ctx.OriginalURL(), "?")
@@ -221,7 +221,7 @@ func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, cont
return httpReq, nil
}
func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
func SetMetaHeaders(ctx fiber.Ctx, meta map[string]string) {
ctx.Response().Header.DisableNormalizing()
for key, val := range meta {
ctx.Response().Header.Set(fmt.Sprintf("x-amz-meta-%s", key), val)
@@ -311,17 +311,17 @@ type CustomHeader struct {
Value string
}
func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
func SetResponseHeaders(ctx fiber.Ctx, headers []CustomHeader) {
for _, header := range headers {
ctx.Set(header.Key, header.Value)
}
}
// Streams the response body by chunks
func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser, bodysize int) {
func StreamResponseBody(ctx fiber.Ctx, rdr io.ReadCloser, bodysize int) {
// SetBodyStream will call Close() on the reader when the stream is done
// since rdr is a ReadCloser
ctx.Context().SetBodyStream(rdr, bodysize)
ctx.RequestCtx().SetBodyStream(rdr, bodysize)
}
func IsValidBucketName(bucket string) bool {
@@ -353,7 +353,7 @@ func includeHeader(hdr string, signedHdrs []string) bool {
})
}
func addRequestHeadersFromCtx(ctx *fiber.Ctx, httpReq *http.Request, signedHdrs []string) error {
func addRequestHeadersFromCtx(ctx fiber.Ctx, httpReq *http.Request, signedHdrs []string) error {
headersNotSigned := []string{}
for key, value := range ctx.Request().Header.All() {
keyStr := string(key)
@@ -424,7 +424,7 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp
return output
}
func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
func ParseObjectAttributes(ctx fiber.Ctx) (map[s3response.ObjectAttributes]struct{}, error) {
attrs := map[s3response.ObjectAttributes]struct{}{}
var err error
for key, value := range ctx.Request().Header.All() {
@@ -463,7 +463,7 @@ type objLockCfg struct {
LegalHoldStatus types.ObjectLockLegalHoldStatus
}
func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
func ParsObjectLockHdrs(ctx fiber.Ctx) (*objLockCfg, error) {
legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold")
objLockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode")
objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date")
@@ -549,7 +549,7 @@ func (cv ChecksumValues) Headers() string {
// ParseCalculatedChecksumHeaders parses and validates x-amz-checksum-x header keys
// e.g x-amz-checksum-crc32, x-amz-checksum-sha256 ...
func ParseCalculatedChecksumHeaders(ctx *fiber.Ctx) (ChecksumValues, error) {
func ParseCalculatedChecksumHeaders(ctx fiber.Ctx) (ChecksumValues, error) {
checksums := ChecksumValues{}
var hdrErr error
@@ -639,7 +639,7 @@ func ParseCalculatedChecksumFields(fields map[string]string) (ChecksumValues, er
// ParseCompleteMpChecksumHeaders parses and validates
// the 'CompleteMultipartUpload' x-amz-checksum-x headers
// by supporting both 'checksum' and 'checksum-<part_length>' formats
func ParseCompleteMpChecksumHeaders(ctx *fiber.Ctx) (ChecksumValues, error) {
func ParseCompleteMpChecksumHeaders(ctx fiber.Ctx) (ChecksumValues, error) {
// first parse/validate 'x-amz-checksum-x' headers
checksums, err := ParseCalculatedChecksumHeaders(ctx)
if err != nil {
@@ -673,7 +673,7 @@ func ParseCompleteMpChecksumHeaders(ctx *fiber.Ctx) (ChecksumValues, error) {
// ParseChecksumHeadersAndSdkAlgo parses/validates 'x-amz-sdk-checksum-algorithm' and
// 'x-amz-checksum-x' precalculated request headers
func ParseChecksumHeadersAndSdkAlgo(ctx *fiber.Ctx) (types.ChecksumAlgorithm, ChecksumValues, error) {
func ParseChecksumHeadersAndSdkAlgo(ctx fiber.Ctx) (types.ChecksumAlgorithm, ChecksumValues, error) {
sdkAlgorithm := types.ChecksumAlgorithm(strings.ToUpper(ctx.Get("X-Amz-Sdk-Checksum-Algorithm")))
err := IsChecksumAlgorithmValid(sdkAlgorithm)
if err != nil {
@@ -862,7 +862,7 @@ func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType
}
// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers
func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) {
func ParseCreateMpChecksumHeaders(ctx fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) {
algo := types.ChecksumAlgorithm(strings.ToUpper(ctx.Get("x-amz-checksum-algorithm")))
if err := IsChecksumAlgorithmValid(algo); err != nil {
return "", "", err
@@ -1071,7 +1071,7 @@ func ValidateCopySource(input string) error {
}
// GetQueryParam returns a pointer to the query parameter value if it exists
func GetQueryParam(ctx *fiber.Ctx, key string) *string {
func GetQueryParam(ctx fiber.Ctx, key string) *string {
value := ctx.Query(key)
if value == "" {
return nil
@@ -1089,9 +1089,9 @@ func ApplyOverride(original, override *string) *string {
// GenerateObjectLocation generates the object location path-styled or host-styled
// depending on the gateway configuration
func GenerateObjectLocation(ctx *fiber.Ctx, virtualDomain, bucket, object string) string {
scheme := ctx.Protocol()
host := ctx.Hostname()
func GenerateObjectLocation(ctx fiber.Ctx, virtualDomain, bucket, object string) string {
scheme := ctx.Scheme()
host := ctx.Host()
// escape the object name
obj := url.PathEscape(object)
@@ -1146,7 +1146,7 @@ func NewTLSListener(network string, address string, getCertificateFunc func(*tls
return tls.NewListener(ln, config), nil
}
func DetectResourceType(ctx *fiber.Ctx) s3err.ResourceType {
func DetectResourceType(ctx fiber.Ctx) s3err.ResourceType {
path := ctx.Path()
if path == "" || path == "/" {
return s3err.ResourceTypeService
s3api/utils/utils_test.go
+2 -2
View File
@@ -28,7 +28,7 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v3"
"github.com/stretchr/testify/assert"
"github.com/valyala/fasthttp"
"github.com/versity/versitygw/backend"
@@ -38,7 +38,7 @@ import (
func TestCreateHttpRequestFromCtx(t *testing.T) {
type args struct {
ctx *fiber.Ctx
ctx fiber.Ctx
}
app := fiber.New()