mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-19 13:46:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: global error refactoring

Browse Source 
Fixes #2123
Fixes #2120
Fixes #2116
Fixes #2111
Fixes #2108
Fixes #2086
Fixes #2085
Fixes #2083
Fixes #2081
Fixes #2080
Fixes #2073
Fixes #2072
Fixes #2071
Fixes #2069
Fixes #2044
Fixes #2043
Fixes #2042
Fixes #2041
Fixes #2040
Fixes #2039
Fixes #2036
Fixes #2035
Fixes #2034
Fixes #2028
Fixes #2020
Fixes #1842
Fixes #1810
Fixes #1780
Fixes #1775
Fixes #1736
Fixes #1705
Fixes #1663
Fixes #1645
Fixes #1583
Fixes #1526
Fixes #1514
Fixes #1493
Fixes #1487
Fixes #959
Fixes #779
Closes #823
Closes #85

Refactor global S3 error handling around structured error types and centralized XML response generation.

All S3 errors now share the common APIError base for the fields every error has: Code, HTTP status code, and Message. Non-traditional errors that need AWS-compatible XML fields now have dedicated typed errors in the s3err package. Each typed error implements the shared S3Error behavior so controllers and middleware can handle errors consistently while still emitting error-specific XML fields.

Add a dedicated InvalidArgumentError type because InvalidArgument is used widely across request validation, auth, copy source handling, object lock validation, multipart validation, and header parsing. The new InvalidArgument path uses explicit InvalidArgErrorCode constants with predefined descriptions and ArgumentName values, keeping call sites readable while preserving the correct InvalidArgument XML shape and optional ArgumentValue.

New structured errors added in s3err:
- `AccessForbiddenError`: Method, ResourceType
- `BadDigestError`: CalculatedDigest, ExpectedDigest
- `BucketError`: BucketName
- `ContentSHA256MismatchError`: ClientComputedContentSHA256, S3ComputedContentSHA256
- `EntityTooLargeError`: ProposedSize, MaxSizeAllowed
- `EntityTooSmallError`: ProposedSize, MinSizeAllowed
- `ExpiredPresignedURLError`: ServerTime, XAmzExpires, Expires
- `InvalidAccessKeyIdError`: AWSAccessKeyId
- `InvalidArgumentError`: Description, ArgumentName, ArgumentValue
- `InvalidChunkSizeError`: Chunk, BadChunkSize
- `InvalidDigestError`: ContentMD5
- `InvalidLocationConstraintError`: LocationConstraint
- `InvalidPartError`: UploadId, PartNumber, ETag
- `InvalidRangeError`: RangeRequested, ActualObjectSize
- `InvalidTagError`: TagKey, TagValue
- `KeyTooLongError`: Size, MaxSizeAllowed
- `MetadataTooLargeError`: Size, MaxSizeAllowed
- `MethodNotAllowedError`: Method, ResourceType, AllowedMethods
- `NoSuchUploadError`: UploadId
- `NoSuchVersionError`: Key, VersionId
- `NotImplementedError`: Header, AdditionalMessage
- `PreconditionFailedError`: Condition
- `RequestTimeTooSkewedError`: RequestTime, ServerTime, MaxAllowedSkewMilliseconds
- `SignatureDoesNotMatchError`: AWSAccessKeyId, StringToSign, SignatureProvided, StringToSignBytes, CanonicalRequest, CanonicalRequestBytes

Fix CompleteMultipartUpload validation in the Azure backend so missing or empty `ETag` values return the appropriate S3 error instead of allowing a gateway panic.

Fix presigned authentication expiration validation to compare server time in `UTC`, matching the `UTC` timestamp used by presigned URL signing.

Add request ID and host ID support across S3 requests. Each request now receives AWS S3-like identifiers, returned in response headers as `x-amz-request-id` and `x-amz-id-2` and included in all XML error responses as RequestId and HostId. The generated ID structure is designed to resemble AWS S3 request IDs and host IDs.

The request signature calculation/validation for streaming uploads was previously delayed until the request body was fully read, both for Authorization header authentication and presigned URLs.
Now, the signature is validated immediately in the authorization middlewares without reading the request body, since the signature calculation itself does not depend on the request body. Instead, only the `x-amz-content-sha256` SHA-256 hash calculation is delayed.
This commit is contained in:
niksis02
2026-05-21 23:49:34 +04:00
parent eade1e3a71
commit 9f786b3c2c
132 changed files with 3511 additions and 1339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3api/admin-server.go
+2
View File
@@ -79,6 +79,8 @@ func NewAdminServer(be backend.Backend, root middlewares.RootUserConfig, region
Format: "${time} | adm | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
}))
}
// initialize requestId middleware
app.Use(middlewares.RequestIDs())
// initialize total requests cap limiter middleware
app.Use(middlewares.RateLimiter(server.maxRequests, nil, l))
s3api/controllers/base.go
+23 -20
View File
@@ -150,6 +150,8 @@ func ProcessHandlers(controller Controller, s3action string, svc *Services, hand
// and metrics. It also handles the error parsing
func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm metrics.Manager) fiber.Handler {
return func(ctx *fiber.Ctx) error {
requestID, hostID := utils.EnsureRequestIDs(ctx)
err := handler(ctx)
if err != nil {
if mm != nil {
@@ -163,18 +165,19 @@ func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm metrics.
ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
serr, ok := err.(s3err.APIError)
if ok {
ctx.Status(serr.HTTPStatusCode)
return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", ""))
if serr, ok := err.(s3err.S3Error); ok {
if mnaErr, ok := serr.(s3err.MethodNotAllowedError); ok && len(mnaErr.AllowedMethods) != 0 {
// for MethodNotAllowed errors, set the 'Allow' header
ctx.Response().Header.Set("Allow", mnaErr.AllowedMethodsString())
}
return ctx.Status(serr.StatusCode()).Send(serr.XMLBody(requestID, hostID))
}
debuglogger.InternalError(err)
ctx.Status(http.StatusInternalServerError)
// If the error is not 's3err.APIError' return 'InternalError'
return ctx.Send(s3err.GetAPIErrorResponse(
s3err.GetAPIError(s3err.ErrInternalError), "", "", ""))
// If the error is not 's3err.S3Error' return 'InternalError'
return ctx.Send(s3err.GetAPIError(s3err.ErrInternalError).XMLBody(requestID, hostID))
}
return ctx.Next()
@@ -188,6 +191,7 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s
// Set the response headers
SetResponseHeaders(ctx, response.Headers)
requestID, hostID := utils.EnsureRequestIDs(ctx)
ensureExposeMetaHeaders(ctx)
opts := response.MetaOpts
@@ -216,18 +220,17 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s
// set content type to application/xml
ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
serr, ok := err.(s3err.APIError)
if ok {
ctx.Status(serr.HTTPStatusCode)
return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", ""))
if serr, ok := err.(s3err.S3Error); ok {
if mnaErr, ok := serr.(s3err.MethodNotAllowedError); ok && len(mnaErr.AllowedMethods) != 0 {
ctx.Response().Header.Set("Allow", mnaErr.AllowedMethodsString())
}
return ctx.Status(serr.StatusCode()).Send(serr.XMLBody(requestID, hostID))
}
debuglogger.InternalError(err)
ctx.Status(http.StatusInternalServerError)
// If the error is not 's3err.APIError' return 'InternalError'
return ctx.Send(s3err.GetAPIErrorResponse(
s3err.GetAPIError(s3err.ErrInternalError), "", "", ""))
// If the error is not 's3err.S3Error' return 'InternalError'
return ctx.Status(http.StatusInternalServerError).Send(s3err.GetAPIError(s3err.ErrInternalError).XMLBody(requestID, hostID))
}
// At this point, the S3 action has succeeded in the backend and
@@ -276,8 +279,9 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s
ObjectSize: opts.ObjectSize,
})
}
return ctx.Status(http.StatusInternalServerError).Send(s3err.GetAPIErrorResponse(
s3err.GetAPIError(s3err.ErrInternalError), "", "", ""))
err := s3err.GetAPIError(s3err.ErrInternalError)
return ctx.Status(err.HTTPStatusCode).Send(err.XMLBody(requestID, hostID))
}
if len(responseBytes) > 0 {
@@ -312,13 +316,12 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s
ObjectSize: opts.ObjectSize,
})
}
ctx.Status(http.StatusInternalServerError)
// set content type to application/xml
ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
return ctx.Send(s3err.GetAPIErrorResponse(
s3err.GetAPIError(s3err.ErrInternalError), "", "", ""))
err := s3err.GetAPIError(s3err.ErrInternalError)
return ctx.Status(err.HTTPStatusCode).Send(err.XMLBody(requestID, hostID))
}
res := make([]byte, 0, msglen)
res = append(res, xmlhdr...)
s3api/controllers/base_test.go
+22 -9
View File
@@ -39,6 +39,9 @@ import (
)
var (
testRequestID = "5MRQJ97RHWJ4FMX9"
testHostID = "eS8nILxNKeV1pNi2Z7Pv6mwC+nuquA2UTBwrBSxGq62e9NZ6f2G9aJPRetuD0/lF3OgqRF7N3GU="
defaultLocals map[utils.ContextKey]any = map[utils.ContextKey]any{
utils.ContextKeyIsRoot: true,
utils.ContextKeyParsedAcl: auth.ACL{
@@ -109,7 +112,7 @@ func testController(t *testing.T, ctrl Controller, resp *Response, expectedErr e
assert.Error(t, err)
switch expectedErr.(type) {
case s3err.APIError:
case s3err.S3Error:
assert.EqualValues(t, expectedErr, err)
default:
assert.ErrorContains(t, err, expectedErr.Error())
@@ -323,7 +326,7 @@ func TestProcessController(t *testing.T) {
},
expected: expected{
status: http.StatusBadRequest,
body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInvalidRequest), "", "", ""),
body: s3err.GetAPIError(s3err.ErrInvalidRequest).XMLBody(testRequestID, testHostID),
},
},
{
@@ -336,7 +339,7 @@ func TestProcessController(t *testing.T) {
},
expected: expected{
status: http.StatusInternalServerError,
body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""),
body: s3err.GetAPIError(s3err.ErrInternalError).XMLBody(testRequestID, testHostID),
},
},
{
@@ -351,7 +354,7 @@ func TestProcessController(t *testing.T) {
},
expected: expected{
status: http.StatusInternalServerError,
body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""),
body: s3err.GetAPIError(s3err.ErrInternalError).XMLBody(testRequestID, testHostID),
},
},
{
@@ -426,7 +429,7 @@ func TestProcessController(t *testing.T) {
},
},
{
name: "large paylod: should return internal error",
name: "large payload: should return internal error",
args: args{
svc: services,
controller: func(ctx *fiber.Ctx) (*Response, error) {
@@ -464,7 +467,7 @@ func TestProcessController(t *testing.T) {
},
},
expected: expected{
body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""),
body: s3err.GetAPIError(s3err.ErrInternalError).XMLBody(testRequestID, testHostID),
status: http.StatusInternalServerError,
},
},
@@ -492,11 +495,15 @@ func TestProcessController(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := fiber.New().AcquireCtx(&fasthttp.RequestCtx{})
utils.ContextKeyRequestID.Set(ctx, testRequestID)
utils.ContextKeyHostID.Set(ctx, testHostID)
err := ProcessController(ctx, tt.args.controller, metrics.ActionAbortMultipartUpload, tt.args.svc)
assert.NoError(t, err)
// check the status
assert.Equal(t, tt.expected.status, ctx.Response().StatusCode())
assert.Equal(t, testRequestID, string(ctx.Response().Header.Peek(utils.HeaderAmzRequestID)))
assert.Equal(t, testHostID, string(ctx.Response().Header.Peek(utils.HeaderAmzID2)))
// check the response headers to be set
if tt.expected.headers != nil {
@@ -556,7 +563,7 @@ func TestProcessHandlers(t *testing.T) {
svc: &Services{},
},
expected: expected{
body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrAccessDenied), "", "", ""),
body: s3err.GetAPIError(s3err.ErrAccessDenied).XMLBody(testRequestID, testHostID),
},
},
{
@@ -591,6 +598,9 @@ func TestProcessHandlers(t *testing.T) {
app := fiber.New()
app.Post("/:bucket/*", func(ctx *fiber.Ctx) error {
utils.ContextKeyRequestID.Set(ctx, testRequestID)
utils.ContextKeyHostID.Set(ctx, testHostID)
// set the request locals
if tt.args.locals != nil {
for key, val := range tt.args.locals {
@@ -654,7 +664,7 @@ func TestWrapMiddleware(t *testing.T) {
logger: &mockAuditLogger{},
},
expected: expected{
body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrAclNotSupported), "", "", ""),
body: s3err.GetAPIError(s3err.ErrAclNotSupported).XMLBody(testRequestID, testHostID),
},
},
{
@@ -665,7 +675,7 @@ func TestWrapMiddleware(t *testing.T) {
},
},
expected: expected{
body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""),
body: s3err.GetAPIError(s3err.ErrInternalError).XMLBody(testRequestID, testHostID),
},
},
}
@@ -675,6 +685,9 @@ func TestWrapMiddleware(t *testing.T) {
app := fiber.New()
app.Post("/:bucket/*", func(ctx *fiber.Ctx) error {
utils.ContextKeyRequestID.Set(ctx, testRequestID)
utils.ContextKeyHostID.Set(ctx, testHostID)
// call the controller by passing the ctx
err := mdlwr(ctx)
assert.NoError(t, err)
s3api/controllers/bucket-get_test.go
+4 -4
View File
@@ -663,7 +663,7 @@ func TestS3ApiController_ListObjectVersions(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetInvalidMaxLimiterErr(utils.LimiterTypeMaxKeys),
err: s3err.GetInvalidArgMaxLimiter(string(utils.LimiterTypeMaxKeys), "invalid"),
},
},
{
@@ -988,7 +988,7 @@ func TestS3ApiController_ListMultipartUploads(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetInvalidMaxLimiterErr(utils.LimiterTypeMaxUploads),
err: s3err.GetInvalidArgMaxLimiter(string(utils.LimiterTypeMaxUploads), "invalid"),
},
},
{
@@ -1095,7 +1095,7 @@ func TestS3ApiController_ListObjectsV2(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetNegativeMaxLimiterErr(utils.LimiterTypeMaxKeys),
err: s3err.GetInvalidArgNegativeMaxLimiter(string(utils.LimiterTypeMaxKeys), "-1"),
},
},
{
@@ -1204,7 +1204,7 @@ func TestS3ApiController_ListObjects(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetInvalidMaxLimiterErr(utils.LimiterTypeMaxKeys),
err: s3err.GetInvalidArgMaxLimiter(string(utils.LimiterTypeMaxKeys), "bla"),
},
},
{
s3api/controllers/bucket-list_test.go
+4 -4
View File
@@ -51,7 +51,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
response: &Response{
MetaOpts: &MetaOptions{},
},
err: s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "-1"),
},
},
{
@@ -66,7 +66,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
response: &Response{
MetaOpts: &MetaOptions{},
},
err: s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "10001"),
},
},
{
@@ -81,7 +81,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
response: &Response{
MetaOpts: &MetaOptions{},
},
err: s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "0"),
},
},
{
@@ -96,7 +96,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
response: &Response{
MetaOpts: &MetaOptions{},
},
err: s3err.GetInvalidMaxLimiterErr("max-buckets"),
err: s3err.GetInvalidArgMaxLimiter("max-buckets", "bla"),
},
},
{
s3api/controllers/bucket-post_test.go
+1 -1
View File
@@ -433,7 +433,7 @@ func TestS3ApiController_POSTObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
err: s3err.GetMetadataTooLargeErr(2053, 2048),
},
},
{
s3api/controllers/bucket-put.go
+6 -11
View File
@@ -17,7 +17,6 @@ package controllers
import (
"encoding/xml"
"errors"
"fmt"
"net/http"
"strings"
@@ -530,7 +529,7 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) {
MetaOpts: &MetaOptions{
BucketOwner: bucketOwner.Access,
},
}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
// both bucket canned ACL and acl grants is not allowed
@@ -566,14 +565,10 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) {
// validate the object ownership value
if ok := utils.IsValidOwnership(objectOwnership); !ok {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: bucketOwner.Access,
},
}, s3err.APIError{
Code: "InvalidArgument",
Description: fmt.Sprintf("Invalid x-amz-object-ownership header: %v", objectOwnership),
HTTPStatusCode: http.StatusBadRequest,
}
MetaOpts: &MetaOptions{
BucketOwner: bucketOwner.Access,
},
}, s3err.GetInvalidArgObjectOwnership(string(objectOwnership))
}
// any bucket ACL(canned, grants) is not allowed with object ownership 'BucketOwnerEnforced'
@@ -610,7 +605,7 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) {
MetaOpts: &MetaOptions{
BucketOwner: bucketOwner.Access,
},
}, s3err.GetAPIError(s3err.ErrInvalidLocationConstraint)
}, s3err.GetInvalidLocationConstraintErr(*body.LocationConstraint)
}
}
}
s3api/controllers/bucket-put_test.go
+5 -9
View File
@@ -737,7 +737,7 @@ func TestS3ApiController_CreateBucket(t *testing.T) {
BucketOwner: adminAcc.Access,
},
},
err: s3err.GetAPIError(s3err.ErrInvalidBucketName),
err: s3err.GetBucketErr(s3err.ErrInvalidBucketName, "invalid_bucket_name"),
},
},
{
@@ -770,7 +770,7 @@ func TestS3ApiController_CreateBucket(t *testing.T) {
response: &Response{
MetaOpts: &MetaOptions{BucketOwner: adminAcc.Access},
},
err: s3err.GetAPIError(s3err.ErrInvalidArgument),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgCannedAcl, "invalid_acl"),
},
},
{
@@ -786,7 +786,7 @@ func TestS3ApiController_CreateBucket(t *testing.T) {
response: &Response{
MetaOpts: &MetaOptions{BucketOwner: adminAcc.Access},
},
err: s3err.GetAPIError(s3err.ErrInvalidLocationConstraint),
err: s3err.GetInvalidLocationConstraintErr("us-west-1"),
},
},
{
@@ -805,11 +805,7 @@ func TestS3ApiController_CreateBucket(t *testing.T) {
BucketOwner: adminAcc.Access,
},
},
err: s3err.APIError{
Code: "InvalidArgument",
Description: "Invalid x-amz-object-ownership header: invalid_ownership",
HTTPStatusCode: http.StatusBadRequest,
},
err: s3err.GetInvalidArgObjectOwnership("invalid_ownership"),
},
},
{
@@ -1110,7 +1106,7 @@ func TestS3ApiController_PutBucketAcl(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidArgument),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgCannedAcl, "invalid_acl"),
},
},
{
s3api/controllers/object-get.go
+2 -2
View File
@@ -457,7 +457,7 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) {
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidPartNumber)
}, s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, ctx.Query("partNumber"))
}
if acceptRange != "" {
@@ -533,7 +533,7 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) {
BucketOwner: parsedAcl.Owner,
Status: status,
},
}, s3err.GetAPIError(s3err.ErrInvalidRange)
}, s3err.GetInvalidRangeErr("", *res.ContentLength)
}
contentLen = int(*res.ContentLength)
}
s3api/controllers/object-get_test.go
+4 -4
View File
@@ -455,7 +455,7 @@ func TestS3ApiController_ListParts(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetInvalidMaxLimiterErr(utils.LimiterTypePartNumberMarker),
err: s3err.GetInvalidArgMaxLimiter(string(utils.LimiterTypePartNumberMarker), "foo"),
},
},
{
@@ -472,7 +472,7 @@ func TestS3ApiController_ListParts(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetNegativeMaxLimiterErr(utils.LimiterTypeMaxParts),
err: s3err.GetInvalidArgNegativeMaxLimiter(string(utils.LimiterTypeMaxParts), "-1"),
},
},
{
@@ -580,7 +580,7 @@ func TestS3ApiController_GetObjectAttributes(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidObjectAttributes),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgObjectAttributes, "invalid_attribute"),
},
},
{
@@ -719,7 +719,7 @@ func TestS3ApiController_GetObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidPartNumber),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, "-2"),
},
},
{
s3api/controllers/object-head.go
+1 -1
View File
@@ -105,7 +105,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) {
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidPartNumber)
}, s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, ctx.Query("partNumber"))
}
if objRange != "" {
s3api/controllers/object-head_test.go
+1 -1
View File
@@ -96,7 +96,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidPartNumber),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, "-4"),
},
},
{
s3api/controllers/object-post_test.go
+30 -12
View File
@@ -250,7 +250,7 @@ func TestS3ApiController_CreateMultipartUpload(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
err: s3err.GetMetadataTooLargeErr(2051, 2048),
},
},
{
@@ -267,7 +267,7 @@ func TestS3ApiController_CreateMultipartUpload(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingObjectLockRetainDate, ""),
},
},
{
@@ -362,17 +362,9 @@ func TestS3ApiController_CompleteMultipartUpload(t *testing.T) {
Parts: []types.CompletedPart{},
})
assert.NoError(t, err)
pn := int32(1)
validMpBody, err := xml.Marshal(s3response.CompleteMultipartUploadRequestBody{
Parts: []types.CompletedPart{
{
PartNumber: &pn,
ETag: utils.GetStringPtr("ETag"),
},
},
})
assert.NoError(t, err)
validMpBody := []byte(`<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Part><PartNumber>1</PartNumber><ETag>ETag</ETag></Part></CompleteMultipartUpload>`)
s3cmdMpBody := []byte("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>ETag</ETag></Part></CompleteMultipartUpload>")
versionId, ETag := "versionId", "mock-ETag"
@@ -569,6 +561,32 @@ func TestS3ApiController_CompleteMultipartUpload(t *testing.T) {
},
},
},
{
name: "successful response with s3cmd request body",
input: testInput{
locals: defaultLocals,
body: s3cmdMpBody,
beRes: s3response.CompleteMultipartUploadResult{ETag: &ETag},
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: s3response.CompleteMultipartUploadResult{
ETag: &ETag,
Location: utils.GetStringPtr("http://example.com/bucket/object"),
},
Headers: map[string]*string{
"x-amz-version-id": &versionId,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventCompleteMultipartUpload,
VersionId: &versionId,
ObjectETag: &ETag,
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s3api/controllers/object-put.go
+4 -4
View File
@@ -261,7 +261,7 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) {
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidPartNumber)
}, s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, ctx.Query("partNumber"))
}
contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
@@ -396,7 +396,7 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) {
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidPartNumber)
}, s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, ctx.Query("partNumber"))
}
preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource())
@@ -566,7 +566,7 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidMetadataDirective)
}, s3err.GetInvalidArgumentErr(s3err.InvalidArgMetadataDirective, string(metaDirective))
}
if taggingDirective != "" && taggingDirective != types.TaggingDirectiveCopy && taggingDirective != types.TaggingDirectiveReplace {
@@ -575,7 +575,7 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidTaggingDirective)
}, s3err.GetInvalidArgumentErr(s3err.InvalidArgTaggingDirective, string(taggingDirective))
}
checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
s3api/controllers/object-put_test.go
+10 -10
View File
@@ -439,7 +439,7 @@ func TestS3ApiController_UploadPart(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidPartNumber),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, "-2"),
},
},
{
@@ -620,7 +620,7 @@ func TestS3ApiController_UploadPartCopy(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "bad%G1"),
},
},
{
@@ -661,7 +661,7 @@ func TestS3ApiController_UploadPartCopy(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidPartNumber),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, "-2"),
},
},
{
@@ -860,7 +860,7 @@ func TestS3ApiController_CopyObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, ""),
},
},
{
@@ -896,7 +896,7 @@ func TestS3ApiController_CopyObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
err: s3err.GetMetadataTooLargeErr(2051, 2048),
},
},
{
@@ -916,7 +916,7 @@ func TestS3ApiController_CopyObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidMetadataDirective),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMetadataDirective, "invalid_metadat_directive"),
},
},
{
@@ -934,7 +934,7 @@ func TestS3ApiController_CopyObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidTaggingDirective),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgTaggingDirective, "invalid_tagging_directive"),
},
},
{
@@ -970,7 +970,7 @@ func TestS3ApiController_CopyObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingObjectLockRetainDate, ""),
},
},
{
@@ -1159,7 +1159,7 @@ func TestS3ApiController_PutObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
err: s3err.GetMetadataTooLargeErr(2059, 2048),
},
},
{
@@ -1177,7 +1177,7 @@ func TestS3ApiController_PutObject(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingObjectLockRetainDate, ""),
},
},
{
s3api/controllers/options.go
+5 -2
View File
@@ -16,6 +16,7 @@ package controllers
import (
"errors"
"net/http"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
@@ -32,6 +33,7 @@ func (s S3ApiController) CORSOptions(ctx *fiber.Ctx) (*Response, error) {
origin := ctx.Get("Origin")
method := auth.CORSHTTPMethod(ctx.Get("Access-Control-Request-Method"))
headers := ctx.Get("Access-Control-Request-Headers")
resourceType := utils.DetectResourceType(ctx)
// Origin is required
if origin == "" {
@@ -67,7 +69,8 @@ func (s S3ApiController) CORSOptions(ctx *fiber.Ctx) (*Response, error) {
if err != nil {
debuglogger.Logf("failed to get bucket cors: %v", err)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchCORSConfiguration)) {
err = s3err.GetAPIError(s3err.ErrCORSIsNotEnabled)
// weirdly s3 always returns BUCKET resource type
err = s3err.GetAccessForbiddenErr(s3err.ErrCORSIsNotEnabled, http.MethodOptions, s3err.ResourceTypeBucket)
debuglogger.Logf("bucket cors is not set: %v", err)
}
return &Response{
@@ -86,7 +89,7 @@ func (s S3ApiController) CORSOptions(ctx *fiber.Ctx) (*Response, error) {
}, err
}
allowConfig, err := corsConfig.IsAllowed(origin, method, parsedHeaders)
allowConfig, err := corsConfig.IsAllowed(origin, method, parsedHeaders, resourceType)
if err != nil {
debuglogger.Logf("cors access forbidden: %v", err)
return &Response{
s3api/controllers/options_test.go
+2 -2
View File
@@ -143,7 +143,7 @@ func TestS3ApiController_CORSOptions(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrCORSIsNotEnabled),
err: s3err.GetAccessForbiddenErr(s3err.ErrCORSIsNotEnabled, http.MethodOptions, s3err.ResourceTypeBucket),
},
},
{
@@ -183,7 +183,7 @@ func TestS3ApiController_CORSOptions(t *testing.T) {
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrCORSForbidden),
err: s3err.GetAccessForbiddenErr(s3err.ErrCORSForbidden, http.MethodOptions, s3err.ResourceTypeObject),
},
},
{
s3api/middlewares/apply-bucket-cors-preflight.go
+1 -1
View File
@@ -43,7 +43,7 @@ func ApplyBucketCORSPreflightFallback(be backend.Backend, fallbackOrigin string)
bucket := ctx.Params("bucket")
_, err := be.GetBucketCors(ctx.Context(), bucket)
if err != nil {
if s3Err, ok := err.(s3err.APIError); ok && (s3Err.Code == "NoSuchCORSConfiguration" || s3Err.Code == "NoSuchBucket") {
if s3Err, ok := err.(s3err.S3Error); ok && (s3Err.BaseError().Code == "NoSuchCORSConfiguration" || s3Err.BaseError().Code == "NoSuchBucket") {
if len(ctx.Response().Header.Peek("Access-Control-Allow-Origin")) == 0 {
ctx.Response().Header.Add("Access-Control-Allow-Origin", fallbackOrigin)
}
s3api/middlewares/apply-bucket-cors.go
+4 -4
View File
@@ -48,8 +48,8 @@ func ApplyBucketCORS(be backend.Backend, fallbackOrigin string) fiber.Handler {
if err != nil {
// If CORS is not configured, S3Error will have code NoSuchCORSConfiguration.
// In this case, we can safely continue. For any other error, we should log it.
s3Err, ok := err.(s3err.APIError)
if ok && (s3Err.Code == "NoSuchCORSConfiguration" || s3Err.Code == "NoSuchBucket") {
s3Err, ok := err.(s3err.S3Error)
if ok && (s3Err.BaseError().Code == "NoSuchCORSConfiguration" || s3Err.BaseError().Code == "NoSuchBucket") {
// Optional global fallback: add Access-Control-Allow-Origin for buckets
// without a specific CORS configuration.
if fallbackOrigin != "" {
@@ -63,7 +63,7 @@ func ApplyBucketCORS(be backend.Backend, fallbackOrigin string) fiber.Handler {
}
return nil
}
if !ok || s3Err.Code != "NoSuchCORSConfiguration" {
if !ok || s3Err.BaseError().Code != "NoSuchCORSConfiguration" {
debuglogger.Logf("failed to get bucket cors for bucket %q: %v", bucket, err)
}
return nil
@@ -99,7 +99,7 @@ func ApplyBucketCORS(be backend.Backend, fallbackOrigin string) fiber.Handler {
return err
}
allowConfig, err := cors.IsAllowed(origin, method, parsedHeaders)
allowConfig, err := cors.IsAllowed(origin, method, parsedHeaders, "")
if err != nil {
// if bucket cors rules doesn't grant access, skip
// and don't add any response headers
s3api/middlewares/authentication.go
+29 -20
View File
@@ -75,7 +75,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
authorization := ctx.Get("Authorization")
if authorization == "" {
return s3err.GetAPIError(s3err.ErrInvalidAuthHeader)
return s3err.GetInvalidArgumentErr(s3err.InvalidArgAuthHeader, authorization)
}
authData, err := utils.ParseAuthorization(authorization)
@@ -91,7 +91,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)
return s3err.GetInvalidAccessKeyIdErr(authData.Access)
}
if err != nil {
return err
@@ -117,29 +117,43 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
if requireContentSha256 && hashPayload == "" {
return s3err.GetAPIError(s3err.ErrMissingContentSha256)
}
if !utils.IsValidSh256PayloadHeader(hashPayload) {
return s3err.GetAPIError(s3err.ErrInvalidSHA256Paylod)
if !utils.IsValidSha256PayloadHeader(hashPayload) {
return s3err.GetInvalidArgumentErr(s3err.InvalidArgSHA256Payload, hashPayload)
}
// the streaming payload type is allowed only in PutObject and UploadPart
// e.g. STREAMING-UNSIGNED-PAYLOAD-TRAILER
if !streamBody && utils.IsStreamingPayload(hashPayload) {
return s3err.GetAPIError(s3err.ErrInvalidSHA256PayloadUsage)
}
if streamBody {
// for streaming PUT actions, authorization is deferred
// until end of stream due to need to get length and
// checksum of the stream to validate authorization
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return utils.NewAuthReader(ctx, r, authData, account.Secret)
})
canonicalString, err := utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength)
if err != nil {
return err
}
if streamBody {
// store the request body stream reader in context locals
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return r
})
// wrap the io.Reader with sha256 hex hash reader, if x-amz-content-sha256
// is the content sha256 - not a special payload type
if !utils.IsSpecialPayload(hashPayload) {
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
var cr io.Reader
cr, err = utils.NewHashReader(r, hashPayload, utils.HashTypeSha256Hex)
return cr
})
if err != nil {
return err
}
}
// wrap the io.Reader with ChunkReader if x-amz-content-sha256
// provide chunk encoding value
if utils.IsStreamingPayload(hashPayload) {
var err error
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
var cr io.Reader
cr, err = utils.NewChunkReader(ctx, r, authData, account.Secret, tdate)
cr, err = utils.NewChunkReader(ctx, r, authData, canonicalString, account.Secret, tdate)
return cr
})
if err != nil {
@@ -156,7 +170,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
// the upload limit for big data actions: PutObject, UploadPart
// is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err
if contentLength > maxObjSizeLimit {
return s3err.GetAPIError(s3err.ErrEntityTooLarge)
return s3err.GetEntityTooLargeErr(contentLength, maxObjSizeLimit)
}
return nil
@@ -169,15 +183,10 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
// Compare the calculated hash with the hash provided
if hashPayload != hexPayload {
return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
return s3err.GetContentSHA256MismatchErr(hashPayload, hexPayload)
}
}
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, false)
if err != nil {
return err
}
return nil
}
}
s3api/middlewares/bucket-object-name-validator.go
+1 -1
View File
@@ -28,7 +28,7 @@ func BucketObjectNameValidator() fiber.Handler {
// check if the provided bucket name is valid
if !utils.IsValidBucketName(bucket) {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
// check if the provided object name is valid
s3api/middlewares/checksum.go
+2 -2
View File
@@ -44,7 +44,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
}
if !isValidMD5(md5sum) {
return s3err.GetAPIError(s3err.ErrInvalidDigest)
return s3err.GetInvalidDigestErr(md5sum)
}
var err error
@@ -67,7 +67,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
var err error
if md5sum != "" {
if !isValidMD5(md5sum) {
return s3err.GetAPIError(s3err.ErrInvalidDigest)
return s3err.GetInvalidDigestErr(md5sum)
}
rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeContentMD5)
s3api/middlewares/object-post-auth.go
+9 -6
View File
@@ -60,7 +60,7 @@ func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string
mediaType, params, err := mime.ParseMediaType(ctx.Get("Content-Type"))
if err != nil || mediaType != fiber.MIMEMultipartForm {
debuglogger.Logf("invalid POST object Content-Type %q: mediaType=%q err=%v", ctx.Get("Content-Type"), mediaType, err)
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
return s3err.GetPreconditionFailedErr(s3err.ConditionPostBucket)
}
boundary := params["boundary"]
@@ -127,14 +127,14 @@ func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string
if algorithm != aws4HMACSHA256 {
debuglogger.Logf("unsupported POST object signing algorithm: %s", algorithm)
return s3err.GetAPIError(s3err.ErrOnlyAws4HmacSha256)
return s3err.GetInvalidArgumentErr(s3err.InvalidArgOnlyAws4HmacSha256, algorithm)
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, amzDate)
if err != nil {
debuglogger.Logf("invalid POST object x-amz-date %q: %v", amzDate, err)
return s3err.GetAPIError(s3err.ErrInvalidDateHeader)
return s3err.GetInvalidArgumentErr(s3err.InvalidArgDateHeader, amzDate)
}
// the signing date can't be older than an hour
@@ -151,13 +151,13 @@ func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string
if region != creds.Region {
debuglogger.Logf("incorrect POST object credential region: got %q want %q", creds.Region, region)
return s3err.PostAuth.IncorrectRegion(region, creds.Region)
return s3err.PostAuth.IncorrectRegion(credentialStr, region, creds.Region)
}
account, err := acct.getAccount(creds.Access)
if err == auth.ErrNoSuchUser {
debuglogger.Logf("POST object access key not found: %s", creds.Access)
return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)
return s3err.GetInvalidAccessKeyIdErr(creds.Access)
}
if err != nil {
debuglogger.Logf("failed to resolve POST object account %q: %v", creds.Access, err)
@@ -174,7 +174,10 @@ func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string
if expectedSig != signatureHex {
debuglogger.Logf("POST object signature mismatch: expected %s got %s", expectedSig, signatureHex)
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
// The String to sign for POST request is the base64 encoded policy
// For POST incorrect signature no canonical request and canonical request bytes are returned
// as the calculation is not based on canonical request string
return s3err.GetSignatureDoesNotMatchErr(account.Access, policyB64, signatureHex, utils.HexBytes(policyB64), "", "")
}
// Mark this request as authenticated so that
s3api/middlewares/object-post-auth_test.go
+2 -2
View File
@@ -52,8 +52,8 @@ func chainHandlers(handlers ...fiber.Handler) fiber.Handler {
func postObjectTestApp(root RootUserConfig, region string, next fiber.Handler) *fiber.App {
app := fiber.New(fiber.Config{
ErrorHandler: func(c *fiber.Ctx, err error) error {
if apiErr, ok := err.(s3err.APIError); ok {
return c.Status(apiErr.HTTPStatusCode).SendString(apiErr.Code)
if s3Err, ok := err.(s3err.S3Error); ok {
return c.Status(s3Err.StatusCode()).Send(s3Err.XMLBody("", ""))
}
return c.Status(500).SendString(err.Error())
},
s3api/middlewares/presign-auth.go
+10 -12
View File
@@ -58,7 +58,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)
return s3err.GetInvalidAccessKeyIdErr(authData.Access)
}
if err != nil {
return err
@@ -75,7 +75,15 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
}
}
err = utils.CheckPresignedSignature(ctx, authData, account.Secret)
if err != nil {
return err
}
if streamBody {
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return r
})
// Content-Length has to be set for data uploads: PutObject, UploadPart
if contentLengthStr == "" {
return s3err.GetAPIError(s3err.ErrMissingContentLength)
@@ -83,18 +91,8 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
// the upload limit for big data actions: PutObject, UploadPart
// is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err
if contentLength > maxObjSizeLimit {
return s3err.GetAPIError(s3err.ErrEntityTooLarge)
return s3err.GetEntityTooLargeErr(contentLength, maxObjSizeLimit)
}
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret)
})
return nil
}
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, streamBody)
if err != nil {
return err
}
return nil
s3api/middlewares/public-bucket.go
+1 -1
View File
@@ -119,7 +119,7 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm
// Compare the calculated hash with the hash provided
if payloadHash != hexPayload {
return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
return s3err.GetContentSHA256MismatchErr(payloadHash, hexPayload)
}
}
s3api/middlewares/rate-limiter.go
+4 -1
View File
@@ -17,6 +17,7 @@ package middlewares
import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
"golang.org/x/sync/semaphore"
@@ -28,6 +29,8 @@ func RateLimiter(limit int, mm metrics.Manager, logger s3log.AuditLogger) fiber.
sem := semaphore.NewWeighted(int64(limit))
return func(ctx *fiber.Ctx) error {
requestID, hostID := utils.EnsureRequestIDs(ctx)
if !sem.TryAcquire(1) {
// limit reached
err := s3err.GetAPIError(s3err.ErrSlowDown)
@@ -42,7 +45,7 @@ func RateLimiter(limit int, mm metrics.Manager, logger s3log.AuditLogger) fiber.
}
ctx.Status(err.HTTPStatusCode)
return ctx.Send(s3err.GetAPIErrorResponse(err, "", "", ""))
return ctx.Send(err.XMLBody(requestID, hostID))
}
defer sem.Release(1)
return ctx.Next()
s3api/middlewares/request-ids.go
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/utils"
)
// RequestIDs sets requestID and hostID in context locals
func RequestIDs() fiber.Handler {
return func(ctx *fiber.Ctx) error {
utils.EnsureRequestIDs(ctx)
return ctx.Next()
}
}
s3api/middlewares/request-ids_test.go
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"net/http"
"net/http/httptest"
"regexp"
"testing"
"github.com/gofiber/fiber/v2"
"github.com/stretchr/testify/assert"
"github.com/versity/versitygw/s3api/utils"
)
func TestRequestIDs(t *testing.T) {
app := fiber.New()
app.Use(RequestIDs())
app.Get("/", func(ctx *fiber.Ctx) error {
assert.NotEmpty(t, utils.RequestID(ctx))
assert.NotEmpty(t, utils.HostID(ctx))
return ctx.SendStatus(http.StatusNoContent)
})
resp, err := app.Test(httptest.NewRequest(http.MethodGet, "/", nil))
assert.NoError(t, err)
requestID := resp.Header.Get(utils.HeaderAmzRequestID)
hostID := resp.Header.Get(utils.HeaderAmzID2)
assert.Regexp(t, regexp.MustCompile(`^[0-9A-Z]{16}$`), requestID)
assert.NotEmpty(t, hostID)
assert.NotEqual(t, requestID, hostID)
}
s3api/router.go
+81 -12
View File
@@ -15,6 +15,8 @@
package s3api
import (
"net/http"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
@@ -156,7 +158,9 @@ func (sa *S3ApiRouter) Init() {
// copy source is not allowed on '/'
sa.app.Get("/", middlewares.MatchHeader("X-Amz-Copy-Source"),
controllers.ProcessHandlers(
ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)),
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
middlewares.ApplyDefaultCORS(sa.corsAllowOrigin),
@@ -467,7 +471,13 @@ func (sa *S3ApiRouter) Init() {
// copy source is not allowed on bucket HEAD operation
bucketRouter.Head("/", middlewares.MatchHeader("X-Amz-Copy-Source"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)), metrics.ActionUndetected, services),
controllers.ProcessHandlers(
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
),
)
bucketRouter.Head("",
@@ -487,7 +497,13 @@ func (sa *S3ApiRouter) Init() {
// copy source is not allowed on bucket DELETE operation
bucketRouter.Delete("/", middlewares.MatchHeader("X-Amz-Copy-Source"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)), metrics.ActionUndetected, services),
controllers.ProcessHandlers(
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
),
)
bucketRouter.Delete("",
@@ -676,7 +692,13 @@ func (sa *S3ApiRouter) Init() {
// copy source is not allowed on bucket GET operation
bucketRouter.Get("/", middlewares.MatchHeader("X-Amz-Copy-Source"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)), metrics.ActionUndetected, services),
controllers.ProcessHandlers(
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
),
)
bucketRouter.Get("",
@@ -1074,7 +1096,13 @@ func (sa *S3ApiRouter) Init() {
bucketRouter.Post("/",
middlewares.MatchHeader("X-Amz-Copy-Source"),
middlewares.MatchQueryArgs("uploadId"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)), metrics.ActionUndetected, services),
controllers.ProcessHandlers(
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
),
)
// DeleteObjects action
@@ -1108,7 +1136,13 @@ func (sa *S3ApiRouter) Init() {
// object HEAD operation is not allowed with copy source
objectRouter.Head("/",
middlewares.MatchHeader("X-Amz-Copy-Source"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)), metrics.ActionUndetected, services),
controllers.ProcessHandlers(
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
),
)
// HeadObject
@@ -1142,7 +1176,13 @@ func (sa *S3ApiRouter) Init() {
// object GET operation is not allowed with copy source
objectRouter.Get("/",
middlewares.MatchHeader("X-Amz-Copy-Source"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)), metrics.ActionUndetected, services),
controllers.ProcessHandlers(
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
),
)
objectRouter.Get("",
@@ -1241,7 +1281,13 @@ func (sa *S3ApiRouter) Init() {
// object DELETE operation is not allowed with copy source
objectRouter.Delete("/",
middlewares.MatchHeader("X-Amz-Copy-Source"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)), metrics.ActionUndetected, services),
controllers.ProcessHandlers(
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
),
)
objectRouter.Delete("",
@@ -1289,7 +1335,13 @@ func (sa *S3ApiRouter) Init() {
objectRouter.Post("/",
middlewares.MatchHeader("X-Amz-Copy-Source"),
middlewares.MatchQueryArgs("uploadId"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrCopySourceNotAllowed)), metrics.ActionUndetected, services),
controllers.ProcessHandlers(
func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, ctx.Get("X-Amz-Copy-Source"))
},
metrics.ActionUndetected,
services,
),
)
objectRouter.Post("",
@@ -1437,13 +1489,28 @@ func (sa *S3ApiRouter) Init() {
// return error if partNumber is used without uploadId
objectRouter.Put("",
middlewares.MatchQueryArgs("partNumber"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrMissingUploadId)), metrics.ActionUndetected, services))
controllers.ProcessHandlers(
ctrl.HandleErrorRoute(s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingUploadId, "partNumber")),
metrics.ActionUndetected,
services,
),
)
// return 'MethodNotAllowed' if uploadId is provided without partNumber
// before the router reaches to 'PutObject'
objectRouter.Put("",
middlewares.MatchQueryArgs("uploadId"),
controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrMethodNotAllowed)), metrics.ActionUndetected, services))
controllers.ProcessHandlers(
ctrl.HandleErrorRoute(
s3err.GetMethodNotAllowedErr(
http.MethodPut,
s3err.ResourceTypeUpload,
[]string{http.MethodDelete, http.MethodPost, http.MethodGet},
),
),
metrics.ActionUndetected,
services,
))
objectRouter.Put("",
middlewares.MatchHeader("X-Amz-Copy-Source"),
@@ -1489,5 +1556,7 @@ func (sa *S3ApiRouter) Init() {
)
// Return MethodNotAllowed for all the unmatched routes
sa.app.All("*", controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrMethodNotAllowed)), metrics.ActionUndetected, services))
sa.app.All("*", controllers.ProcessHandlers(func(ctx *fiber.Ctx) (*controllers.Response, error) {
return &controllers.Response{}, s3err.GetMethodNotAllowedErr(ctx.Method(), s3err.ResourceTypeService, nil)
}, metrics.ActionUndetected, services))
}
s3api/server.go
+8 -4
View File
@@ -131,6 +131,10 @@ func New(
Format: "${time} | vgw | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
}))
}
// initialize requestId middleware
app.Use(middlewares.RequestIDs())
// Set up health endpoint if specified
if server.health != "" {
app.Get(server.health, func(ctx *fiber.Ctx) error {
@@ -379,6 +383,8 @@ func stackTraceHandler(ctx *fiber.Ctx, e any) {
// globalErrorHandler catches the errors before reaching to
// the handlers and any system panics
func globalErrorHandler(ctx *fiber.Ctx, er error) error {
requestID, hostID := utils.EnsureRequestIDs(ctx)
// set content type to application/xml
ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML)
@@ -406,8 +412,7 @@ func globalErrorHandler(ctx *fiber.Ctx, er error) error {
// which is a malfoedmed one. Return a BadRequest in this case
debuglogger.Logf("failed to parse the http request")
err := s3err.GetAPIError(s3err.ErrCannotParseHTTPRequest)
ctx.Status(err.HTTPStatusCode)
return ctx.Send(s3err.GetAPIErrorResponse(err, "", "", ""))
return ctx.Status(err.StatusCode()).Send(err.XMLBody(requestID, hostID))
}
}
@@ -417,6 +422,5 @@ func globalErrorHandler(ctx *fiber.Ctx, er error) error {
ctx.Status(http.StatusInternalServerError)
return ctx.Send(s3err.GetAPIErrorResponse(
s3err.GetAPIError(s3err.ErrInternalError), "", "", ""))
return ctx.Send(s3err.GetAPIError(s3err.ErrInternalError).XMLBody(requestID, hostID))
}
s3api/utils/auth-reader.go
+32 -89
View File
@@ -18,9 +18,7 @@ import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"strings"
"time"
@@ -39,77 +37,15 @@ const (
yyyymmdd = "20060102"
)
// AuthReader is an io.Reader that validates the request authorization
// once the underlying reader returns io.EOF. This is needed for streaming
// data requests where the data size and checksum are not known until
// the data is completely read.
type AuthReader struct {
ctx *fiber.Ctx
auth AuthData
secret string
size int
r *HashReader
}
func HexBytes(s string) string {
b := []byte(s) // raw UTF-8 bytes
// NewAuthReader initializes an io.Reader that will verify the request
// v4 auth when the underlying reader returns io.EOF. This postpones the
// authorization check until the reader is consumed. So it is important that
// the consumer of this reader checks for the auth errors while reading.
func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string) *AuthReader {
var hr *HashReader
hashPayload := ctx.Get("X-Amz-Content-Sha256")
if !IsSpecialPayload(hashPayload) {
hr, _ = NewHashReader(r, "", HashTypeSha256Hex)
} else {
hr, _ = NewHashReader(r, "", HashTypeNone)
parts := make([]string, len(b))
for i, v := range b {
parts[i] = fmt.Sprintf("%02x", v)
}
return &AuthReader{
ctx: ctx,
r: hr,
auth: auth,
secret: secret,
}
}
// Read allows *AuthReader to be used as an io.Reader
func (ar *AuthReader) Read(p []byte) (int, error) {
n, err := ar.r.Read(p)
ar.size += n
if errors.Is(err, io.EOF) {
verr := ar.validateSignature()
if verr != nil {
return n, verr
}
}
return n, err
}
func (ar *AuthReader) validateSignature() error {
date := ar.ctx.Get("X-Amz-Date")
if date == "" {
return s3err.GetAPIError(s3err.ErrMissingDateHeader)
}
hashPayload := ar.ctx.Get("X-Amz-Content-Sha256")
if !IsSpecialPayload(hashPayload) {
hexPayload := ar.r.Sum()
// Compare the calculated hash with the hash provided
if hashPayload != hexPayload {
return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
}
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return s3err.GetAPIError(s3err.ErrMissingDateHeader)
}
return CheckValidSignature(ar.ctx, ar.auth, ar.secret, hashPayload, tdate, int64(ar.size), true)
return strings.Join(parts, " ")
}
const (
@@ -117,18 +53,18 @@ const (
)
// CheckValidSignature validates the ctx v4 auth signature
func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64, streamBody bool) error {
func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64) (string, error) {
signedHdrs := strings.Split(auth.SignedHeaders, ";")
// Create a new http request instance from fasthttp request
req, err := createHttpRequestFromCtx(ctx, signedHdrs, contentLen, streamBody)
req, err := createHttpRequestFromCtx(ctx, signedHdrs, contentLen)
if err != nil {
return fmt.Errorf("create http request from context: %w", err)
return "", fmt.Errorf("create http request from context: %w", err)
}
signer := v4.NewSigner()
signErr := signer.SignHTTP(req.Context(),
signMeta, err := signer.SignHTTP(req.Context(),
aws.Credentials{
AccessKeyID: auth.Access,
SecretAccessKey: secret,
@@ -141,20 +77,27 @@ func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string,
options.Logger = logging.NewStandardLogger(os.Stderr)
}
})
if signErr != nil {
return fmt.Errorf("sign generated http request: %w", err)
if err != nil {
return "", fmt.Errorf("sign generated http request: %w", err)
}
genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
if err != nil {
return err
return "", err
}
if auth.Signature != genAuth.Signature {
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
return "", s3err.GetSignatureDoesNotMatchErr(
auth.Access,
signMeta.StringToSign,
auth.Signature,
HexBytes(signMeta.StringToSign),
signMeta.CanonicalString,
HexBytes(signMeta.CanonicalString),
)
}
return nil
return signMeta.CanonicalString, nil
}
// AuthData is the parsed authorization data from the header
@@ -187,7 +130,7 @@ func ParseAuthorization(authorization string) (AuthData, error) {
}
if len(authParts) < 2 {
return a, s3err.GetAPIError(s3err.ErrInvalidAuthHeader)
return a, s3err.GetInvalidArgumentErr(s3err.InvalidArgAuthHeader, authorization)
}
algo := authParts[0]
@@ -196,7 +139,7 @@ func ParseAuthorization(authorization string) (AuthData, error) {
return a, s3err.GetAPIError(s3err.ErrUnsupportedAuthorizationMechanism)
}
if algo != "AWS4-HMAC-SHA256" {
return a, s3err.GetAPIError(s3err.ErrUnsupportedAuthorizationType)
return a, s3err.GetInvalidArgumentErr(s3err.InvalidArgAuthorizationType, algo)
}
kvData := authParts[1]
@@ -263,26 +206,26 @@ type CredentialsScope struct {
}
type CredsError interface {
MalformedCredential() s3err.APIError
IncorrectService(string) s3err.APIError
IncorrectTerminal(string) s3err.APIError
InvalidDateFormat(string) s3err.APIError
MalformedCredential(string) s3err.S3Error
IncorrectService(string, string) s3err.S3Error
IncorrectTerminal(string, string) s3err.S3Error
InvalidDateFormat(string, string) s3err.S3Error
}
func ParseCredentials(input string, errHandler CredsError) (*CredentialsScope, error) {
creds := strings.Split(input, "/")
if len(creds) != 5 {
return nil, errHandler.MalformedCredential()
return nil, errHandler.MalformedCredential(input)
}
if creds[3] != "s3" {
return nil, errHandler.IncorrectService(creds[3])
return nil, errHandler.IncorrectService(input, creds[3])
}
if creds[4] != "aws4_request" {
return nil, errHandler.IncorrectTerminal(creds[4])
return nil, errHandler.IncorrectTerminal(input, creds[4])
}
_, err := time.Parse(yyyymmdd, creds[1])
if err != nil {
return nil, errHandler.InvalidDateFormat(creds[1])
return nil, errHandler.InvalidDateFormat(input, creds[1])
}
return &CredentialsScope{
Access: creds[0],
s3api/utils/auth_test.go
+2 -2
View File
@@ -92,7 +92,7 @@ func Test_Client_UserAgent(t *testing.T) {
}
app.Get("/", func(c *fiber.Ctx) error {
req, err := createHttpRequestFromCtx(c, signedHdrs, int64(c.Request().Header.ContentLength()), true)
req, err := createHttpRequestFromCtx(c, signedHdrs, int64(c.Request().Header.ContentLength()))
if err != nil {
t.Fatal(err)
}
@@ -102,7 +102,7 @@ func Test_Client_UserAgent(t *testing.T) {
signer := v4.NewSigner()
signErr := signer.SignHTTP(req.Context(),
_, signErr := signer.SignHTTP(req.Context(),
aws.Credentials{
AccessKeyID: access,
SecretAccessKey: secret,
s3api/utils/chunk-reader.go
+9 -9
View File
@@ -116,15 +116,15 @@ func IsSpecialPayload(str string) bool {
return specialValues[payloadType(str)]
}
// IsValidSh256PayloadHeader checks if the provided x-amz-content-sha256
// paylod header is valid special paylod type or a valid sh256 hash
func IsValidSh256PayloadHeader(value string) bool {
// IsValidSha256PayloadHeader checks if the provided x-amz-content-sha256
// payload header is valid special payload type or a valid sh256 hash
func IsValidSha256PayloadHeader(value string) bool {
// empty header is valid
if value == "" {
return true
}
// special values are valid
if specialValues[payloadType(value)] {
if IsSpecialPayload(value) {
return true
}
@@ -162,7 +162,7 @@ func IsUnsignedPaylod(hash string) bool {
return hash == string(payloadTypeUnsigned)
}
// IsChunkEncoding checks for streaming/unsigned authorization types
// IsStreamingPayload checks for streaming/unsigned authorization types
func IsStreamingPayload(str string) bool {
pt := payloadType(str)
return pt == payloadTypeStreamingUnsignedTrailer ||
@@ -186,13 +186,13 @@ func ParseDecodedContentLength(ctx *fiber.Ctx) (int64, error) {
if decContLength > maxObjSizeLimit {
debuglogger.Logf("the object size exceeds the allowed limit: (size): %v, (limit): %v", decContLength, int64(maxObjSizeLimit))
return 0, s3err.GetAPIError(s3err.ErrEntityTooLarge)
return 0, s3err.GetEntityTooLargeErr(decContLength, maxObjSizeLimit)
}
return decContLength, nil
}
func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, secret string, date time.Time) (io.Reader, error) {
func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, canonicalString, secret string, date time.Time) (io.Reader, error) {
cLength, err := ParseDecodedContentLength(ctx)
if err != nil {
return nil, err
@@ -214,9 +214,9 @@ func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, secret strin
case payloadTypeStreamingUnsignedTrailer:
return NewUnsignedChunkReader(r, checksumType, cLength)
case payloadTypeStreamingSignedTrailer:
return NewSignedChunkReader(r, authdata, secret, date, checksumType, true, cLength)
return NewSignedChunkReader(r, authdata, canonicalString, secret, date, checksumType, true, cLength)
case payloadTypeStreamingSigned:
return NewSignedChunkReader(r, authdata, secret, date, "", false, cLength)
return NewSignedChunkReader(r, authdata, canonicalString, secret, date, "", false, cLength)
// return not supported for:
// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
// - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
s3api/utils/chunk-reader_test.go
+3 -3
View File
@@ -16,7 +16,7 @@ package utils
import "testing"
func TestIsValidSh256PayloadHeader(t *testing.T) {
func TestIsValidSha256PayloadHeader(t *testing.T) {
tests := []struct {
name string
hash string
@@ -35,8 +35,8 @@ func TestIsValidSh256PayloadHeader(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsValidSh256PayloadHeader(tt.hash); got != tt.want {
t.Errorf("IsValidSh256PayloadHeader() = %v, want %v", got, tt.want)
if got := IsValidSha256PayloadHeader(tt.hash); got != tt.want {
t.Errorf("IsValidSha256PayloadHeader() = %v, want %v", got, tt.want)
}
})
}
s3api/utils/context-keys.go
+2 -16
View File
@@ -38,24 +38,10 @@ const (
ContextKeyStack ContextKey = "stack"
ContextKeyBucketOwner ContextKey = "bucket-owner"
ContextKeyObjectPostResult ContextKey = "object-post-result"
ContextKeyRequestID ContextKey = "request-id"
ContextKeyHostID ContextKey = "host-id"
)
func (ck ContextKey) Values() []ContextKey {
return []ContextKey{
ContextKeyRegion,
ContextKeyStartTime,
ContextKeyIsRoot,
ContextKeyRootAccessKey,
ContextKeyAccount,
ContextKeyAuthenticated,
ContextKeyPublicBucket,
ContextKeyParsedAcl,
ContextKeySkipResBodyLog,
ContextKeyBodyReader,
ContextKeyBucketOwner,
}
}
func (ck ContextKey) Set(ctx *fiber.Ctx, val any) {
ctx.Locals(string(ck), val)
}
s3api/utils/csum-reader.go
+11 -2
View File
@@ -136,7 +136,7 @@ func (hr *HashReader) Read(p []byte) (int, error) {
case HashTypeContentMD5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrBadDigest)
return n, s3err.GetBadDigestErr(sum, hr.base64ToHex(hr.sum))
}
case HashTypeMd5:
sum := hr.Sum()
@@ -146,7 +146,7 @@ func (hr *HashReader) Read(p []byte) (int, error) {
case HashTypeSha256Hex:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
return n, s3err.GetContentSHA256MismatchErr(hr.sum, sum)
}
case HashTypeCRC32:
sum := hr.Sum()
@@ -200,6 +200,15 @@ func (hr *HashReader) Read(p []byte) (int, error) {
return n, readerr
}
func (hr *HashReader) base64ToHex(s string) string {
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return ""
}
return hex.EncodeToString(b)
}
func (hr *HashReader) SetReader(r io.Reader) {
hr.r = r
}
s3api/utils/multipart_reader.go
+2 -1
View File
@@ -224,7 +224,8 @@ func (mp *MultipartParser) readFieldValue() (string, error) {
case finalBoundaryLine:
debuglogger.Logf("multipart POST ended before file part was found")
return "", s3err.GetAPIError(s3err.ErrPOSTFileRequired)
// S3 returns '0' as ArgumentValue
return "", s3err.GetInvalidArgumentErr(s3err.InvalidArgPOSTFileRequired, "0")
default:
buf.Write(raw)
s3api/utils/multipart_reader_test.go
+1 -1
View File
@@ -292,7 +292,7 @@ func TestMultipartParserParseErrors(t *testing.T) {
"value\r\n",
"--abc--\r\n",
}, ""),
want: s3err.GetAPIError(s3err.ErrPOSTFileRequired),
want: s3err.GetInvalidArgumentErr(s3err.InvalidArgPOSTFileRequired, "0"),
},
{
name: "line without crlf terminator",
s3api/utils/presign-auth-reader.go
+17 -46
View File
@@ -15,9 +15,7 @@
package utils
import (
"errors"
"fmt"
"io"
"net/url"
"os"
"strconv"
@@ -25,9 +23,9 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go/logging"
"github.com/gofiber/fiber/v2"
v4 "github.com/versity/versitygw/aws/signer/v4"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
)
@@ -39,42 +37,8 @@ const (
algoECDSA string = "AWS4-ECDSA-P256-SHA256"
)
// PresignedAuthReader is an io.Reader that validates presigned request authorization
// once the underlying reader returns io.EOF. This is needed for streaming
// data requests where the data size is not known until
// the data is completely read.
type PresignedAuthReader struct {
ctx *fiber.Ctx
auth AuthData
secret string
r io.Reader
}
func NewPresignedAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string) *PresignedAuthReader {
return &PresignedAuthReader{
ctx: ctx,
r: r,
auth: auth,
secret: secret,
}
}
// Read allows *PresignedAuthReader to be used as an io.Reader
func (pr *PresignedAuthReader) Read(p []byte) (int, error) {
n, err := pr.r.Read(p)
if errors.Is(err, io.EOF) {
cerr := CheckPresignedSignature(pr.ctx, pr.auth, pr.secret, true)
if cerr != nil {
return n, cerr
}
}
return n, err
}
// CheckPresignedSignature validates presigned request signature
func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, streamBody bool) error {
func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string) error {
signedHdrs := strings.Split(auth.SignedHeaders, ";")
var contentLength int64
@@ -88,7 +52,7 @@ func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, strea
}
// Create a new http request instance from fasthttp request
req, err := createPresignedHttpRequestFromCtx(ctx, signedHdrs, contentLength, streamBody)
req, err := createPresignedHttpRequestFromCtx(ctx, signedHdrs, contentLength)
if err != nil {
return fmt.Errorf("create http request from context: %w", err)
}
@@ -96,10 +60,10 @@ func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, strea
date, _ := time.Parse(iso8601Format, auth.Date)
signer := v4.NewSigner()
uri, _, signErr := signer.PresignHTTP(ctx.Context(), aws.Credentials{
uri, _, signMeta, signErr := signer.PresignHTTP(ctx.Context(), aws.Credentials{
AccessKeyID: auth.Access,
SecretAccessKey: secret,
}, req, unsignedPayload, service, auth.Region, date, func(options *v4.SignerOptions) {
}, req, unsignedPayload, service, auth.Region, date, signedHdrs, func(options *v4.SignerOptions) {
options.DisableURIPathEscaping = true
if debuglogger.IsDebugEnabled() {
options.LogSigning = true
@@ -117,7 +81,14 @@ func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, strea
signature := urlParts.Query().Get("X-Amz-Signature")
if signature != auth.Signature {
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
return s3err.GetSignatureDoesNotMatchErr(
auth.Access,
signMeta.StringToSign,
auth.Signature,
HexBytes(signMeta.StringToSign),
signMeta.CanonicalString,
HexBytes(signMeta.CanonicalString),
)
}
return nil
@@ -218,11 +189,11 @@ func validateExpiration(str string, date time.Time) error {
return s3err.QueryAuthErrors.ExpiresTooLarge()
}
now := time.Now()
passed := int(now.Sub(date).Seconds())
now := time.Now().UTC()
expiresAt := date.Add(time.Duration(exp) * time.Second)
if passed > exp {
return s3err.GetAPIError(s3err.ErrExpiredPresignRequest)
if expiresAt.Before(now) {
return s3err.GetExpiredPresignedURLError(exp, expiresAt.Format(time.RFC3339), now.Format(time.RFC3339))
}
return nil
s3api/utils/request_ids.go
+114
View File
@@ -0,0 +1,114 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"crypto/rand"
"encoding/base64"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/debuglogger"
)
const (
HeaderAmzRequestID = "x-amz-request-id"
HeaderAmzID2 = "x-amz-id-2"
s3RequestIDAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
s3RequestIDLength = 16
s3HostIDBytes = 65
)
// NewS3RequestID returns a request ID, for example
// "5MRQJ97RHWJ4FMX9".
func NewS3RequestID() string {
return randomBase36(s3RequestIDLength)
}
func randomBase36(length int) string {
const maxUnbiasedByte = byte(252) // 36 * 7
out := make([]byte, length)
buf := make([]byte, length*2)
for i := 0; i < length; {
mustReadRandom(buf)
for _, b := range buf {
if b >= maxUnbiasedByte {
continue
}
out[i] = s3RequestIDAlphabet[int(b)%len(s3RequestIDAlphabet)]
i++
if i == length {
break
}
}
}
return string(out)
}
// NewS3HostID generates a new s3-style host ID
func NewS3HostID() string {
b := make([]byte, s3HostIDBytes)
mustReadRandom(b)
return base64.StdEncoding.EncodeToString(b)
}
// EnsureRequestIDs makes sure the request-local IDs exist and are present
// in the response headers. Existing local values are reused so headers and XML
// bodies stay consistent throughout the request.
func EnsureRequestIDs(ctx *fiber.Ctx) (requestID, hostID string) {
requestID = RequestID(ctx)
if requestID == "" {
requestID = NewS3RequestID()
ContextKeyRequestID.Set(ctx, requestID)
}
hostID = HostID(ctx)
if hostID == "" {
hostID = NewS3HostID()
ContextKeyHostID.Set(ctx, hostID)
}
ctx.Response().Header.Set(HeaderAmzRequestID, requestID)
ctx.Response().Header.Set(HeaderAmzID2, hostID)
return requestID, hostID
}
func RequestID(ctx *fiber.Ctx) string {
requestID, _ := ContextKeyRequestID.Get(ctx).(string)
if requestID != "" {
return requestID
}
return string(ctx.Response().Header.Peek(HeaderAmzRequestID))
}
func HostID(ctx *fiber.Ctx) string {
hostID, _ := ContextKeyHostID.Get(ctx).(string)
if hostID != "" {
return hostID
}
return string(ctx.Response().Header.Peek(HeaderAmzID2))
}
func mustReadRandom(b []byte) {
if _, err := rand.Read(b); err != nil {
debuglogger.Logf("randomize ID bytes: %v", err)
}
}
s3api/utils/request_ids_test.go
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"encoding/base64"
"regexp"
"testing"
"github.com/gofiber/fiber/v2"
"github.com/stretchr/testify/assert"
"github.com/valyala/fasthttp"
)
func TestNewS3RequestID(t *testing.T) {
id := NewS3RequestID()
assert.Regexp(t, regexp.MustCompile(`^[0-9A-Z]{16}$`), id)
}
func TestNewS3HostID(t *testing.T) {
id := NewS3HostID()
decoded, err := base64.StdEncoding.DecodeString(id)
assert.NoError(t, err)
assert.Len(t, decoded, s3HostIDBytes)
}
func TestEnsureRequestIDs(t *testing.T) {
ctx := fiber.New().AcquireCtx(&fasthttp.RequestCtx{})
requestID, hostID := EnsureRequestIDs(ctx)
requestIDAgain, hostIDAgain := EnsureRequestIDs(ctx)
assert.Equal(t, requestID, requestIDAgain)
assert.Equal(t, hostID, hostIDAgain)
assert.Equal(t, requestID, RequestID(ctx))
assert.Equal(t, hostID, HostID(ctx))
assert.Equal(t, requestID, string(ctx.Response().Header.Peek(HeaderAmzRequestID)))
assert.Equal(t, hostID, string(ctx.Response().Header.Peek(HeaderAmzID2)))
}
s3api/utils/signed-chunk-reader.go
+37 -33
View File
@@ -58,46 +58,50 @@ var (
// ChunkReader reads from chunked upload request body, and returns
// object data stream
type ChunkReader struct {
r io.Reader
signingKey []byte
prevSig string
parsedSig string
chunkDataLeft int64
trailer checksumType
trailerSig string
parsedChecksum string
stash []byte
chunkHash hash.Hash
checksumHash hash.Hash
isEOF bool
isFirstHeader bool
region string
date time.Time
requireTrailer bool
chunkSizes []int64
cLength int64
dataRead int64
r io.Reader
signingKey []byte
prevSig string
parsedSig string
canonicalString string
accessKey string
chunkDataLeft int64
trailer checksumType
trailerSig string
parsedChecksum string
stash []byte
chunkHash hash.Hash
checksumHash hash.Hash
isEOF bool
isFirstHeader bool
region string
date time.Time
requireTrailer bool
chunkSizes []int64
cLength int64
dataRead int64
}
// NewChunkReader reads from request body io.Reader and parses out the
// chunk metadata in stream. The headers are validated for proper signatures.
// Reading from the chunk reader will read only the object data stream
// without the chunk headers/trailers.
func NewSignedChunkReader(r io.Reader, authdata AuthData, secret string, date time.Time, chType checksumType, requireTrailer bool, cLength int64) (io.Reader, error) {
func NewSignedChunkReader(r io.Reader, authdata AuthData, canonicalString, secret string, date time.Time, chType checksumType, requireTrailer bool, cLength int64) (io.Reader, error) {
chRdr := &ChunkReader{
r: r,
signingKey: getSigningKey(secret, authdata.Region, date),
// the authdata.Signature is validated in the auth-reader,
// so we can use that here without any other checks
prevSig: authdata.Signature,
chunkHash: sha256.New(),
isFirstHeader: true,
date: date,
region: authdata.Region,
trailer: chType,
requireTrailer: requireTrailer,
chunkSizes: []int64{},
cLength: cLength,
prevSig: authdata.Signature,
canonicalString: canonicalString,
accessKey: authdata.Access,
chunkHash: sha256.New(),
isFirstHeader: true,
date: date,
region: authdata.Region,
trailer: chType,
requireTrailer: requireTrailer,
chunkSizes: []int64{},
cLength: cLength,
}
if chType != "" {
@@ -224,7 +228,7 @@ func (cr *ChunkReader) verifyTrailerSignature() error {
if sig != cr.trailerSig {
debuglogger.Logf("incorrect trailing signature: (calculated): %v, (got): %v", sig, cr.trailerSig)
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
return s3err.GetSignatureDoesNotMatchErr(cr.accessKey, strToSign, cr.trailerSig, HexBytes(strToSign), cr.canonicalString, HexBytes(cr.canonicalString))
}
return nil
@@ -251,7 +255,7 @@ func (cr *ChunkReader) checkSignature() error {
if cr.prevSig != cr.parsedSig {
debuglogger.Logf("incorrect signature: (calculated): %v, (got) %v", cr.prevSig, cr.parsedSig)
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
return s3err.GetSignatureDoesNotMatchErr(cr.accessKey, sigstr, cr.parsedSig, HexBytes(sigstr), cr.canonicalString, HexBytes(cr.canonicalString))
}
cr.parsedSig = ""
return nil
@@ -327,7 +331,7 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
if (chunkSize + int64(n)) > math.MaxInt {
debuglogger.Logf("exceeding the limit of maximum integer allowed: (value): %v, (limit): %v", chunkSize+int64(n), math.MaxInt)
return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
return 0, s3err.GetAPIError(s3err.ErrIncompleteBody)
}
return n + int(chunkSize), err
}
@@ -541,7 +545,7 @@ func (cr *ChunkReader) parseChunkSize(rdr *bufio.Reader, header []byte) (int64,
}
if !cr.isValidChunkSize(chunkSize) {
return 0, s3err.GetAPIError(s3err.ErrInvalidChunkSize)
return 0, s3err.GetInvalidChunkSizeErr(len(cr.chunkSizes)+1, chunkSize)
}
return chunkSize, nil
s3api/utils/unsigned-chunk-reader_test.go
+3 -3
View File
@@ -167,11 +167,11 @@ func requireAPIErrorCode(t *testing.T, err error, code string) {
if err == nil {
t.Fatalf("expected %s error, got nil", code)
}
var apiErr s3err.APIError
var apiErr s3err.S3Error
if !errors.As(err, &apiErr) {
t.Fatalf("expected APIError, got %T: %v", err, err)
}
if apiErr.Code != code {
t.Fatalf("APIError code = %q, want %q", apiErr.Code, code)
if apiErr.BaseError().Code != code {
t.Fatalf("APIError code = %q, want %q", apiErr.BaseError().Code, code)
}
}
s3api/utils/utils.go
+52 -44
View File
@@ -15,7 +15,6 @@
package utils
import (
"bytes"
"crypto/tls"
"encoding/base64"
"encoding/xml"
@@ -94,7 +93,7 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (map[string]string, error)
if metadataSize > maxMetadataSize {
debuglogger.Logf("total meta headers size exceeded the maximum allowed: (size): %v, (max): %v", metadataSize, maxMetadataSize)
return nil, s3err.GetAPIError(s3err.ErrMetadataTooLarge)
return nil, s3err.GetMetadataTooLargeErr(metadataSize, maxMetadataSize)
}
}
}
@@ -115,7 +114,7 @@ func ExtractMetadataFromFields(fields map[string]string) (map[string]string, err
metadataSize += len(trimmedKey) + len(value)
if metadataSize > maxMetadataSize {
debuglogger.Logf("total meta headers size exceeded the maximum allowed: (size): %v, (max): %v", metadataSize, maxMetadataSize)
return nil, s3err.GetAPIError(s3err.ErrMetadataTooLarge)
return nil, s3err.GetMetadataTooLargeErr(metadataSize, maxMetadataSize)
}
metadata[trimmedKey] = value
@@ -124,18 +123,12 @@ func ExtractMetadataFromFields(fields map[string]string) (map[string]string, err
return metadata, nil
}
func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64, streamBody bool) (*http.Request, error) {
func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
req := ctx.Request()
var body io.Reader
if streamBody {
body = req.BodyStream()
} else {
body = bytes.NewReader(req.Body())
}
uri := ctx.OriginalURL()
httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
httpReq, err := http.NewRequest(string(req.Header.Method()), uri, nil)
if err != nil {
return nil, errors.New("error in creating an http request")
}
@@ -179,14 +172,8 @@ var (
}
)
func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64, streamBody bool) (*http.Request, error) {
func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
req := ctx.Request()
var body io.Reader
if streamBody {
body = req.BodyStream()
} else {
body = bytes.NewReader(req.Body())
}
uri, _, _ := strings.Cut(ctx.OriginalURL(), "?")
isFirst := true
@@ -204,7 +191,7 @@ func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, cont
}
}
httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
httpReq, err := http.NewRequest(string(req.Header.Method()), uri, nil)
if err != nil {
return nil, errors.New("error in creating an http request")
}
@@ -279,14 +266,14 @@ func ParseMaxLimiter(limiter string, lt LimiterType) (int32, error) {
lt = LimiterTypeMaxKeys
}
debuglogger.Logf("invalid %s provided: %s\n", lt, limiter)
return 0, s3err.GetInvalidMaxLimiterErr(string(lt))
return 0, s3err.GetInvalidArgMaxLimiter(string(lt), limiter)
}
// max-buckets has distinct range rules and errors.
if lt == LimiterTypeMaxBuckets {
if num < 1 || num > int64(defaultMaxBuckets) {
debuglogger.Logf("invalid max-buckets: %v", num)
return 0, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets)
return 0, s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, limiter)
}
return int32(num), nil
}
@@ -302,10 +289,10 @@ func ParseMaxLimiter(limiter string, lt LimiterType) (int32, error) {
// versions_max_keys uses the MaxKeys negative error.
if lt == LimiterTypeVersionsMaxKeys {
return 0, s3err.GetAPIError(s3err.ErrNegativeMaxKeys)
return 0, s3err.GetInvalidArgumentErr(s3err.InvalidArgNegativeMaxKeys, limiter)
}
return 0, s3err.GetNegativeMaxLimiterErr(string(lt))
return 0, s3err.GetInvalidArgNegativeMaxLimiter(string(lt), limiter)
}
// Clamp excessive limiters to defaultMaxLimiter.
@@ -367,7 +354,7 @@ func includeHeader(hdr string, signedHdrs []string) bool {
// expiration time window
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationTimeStamp
const timeExpirationSec = 15 * 60
const timeExpirationSec = 15 * 60 // seconds
func ValidateDate(date time.Time) error {
now := time.Now().UTC()
@@ -375,7 +362,7 @@ func ValidateDate(date time.Time) error {
// Checks the dates difference to be within allotted window
if diff > timeExpirationSec || diff < -timeExpirationSec {
return s3err.GetAPIError(s3err.ErrRequestTimeTooSkewed)
return s3err.GetRequestTimeTooSkewedErr(date.Format(iso8601Format), now.Format(time.RFC3339), timeExpirationSec*1000)
}
return nil
@@ -427,7 +414,7 @@ func ParseObjectAttributes(ctx *fiber.Ctx) (map[s3response.ObjectAttributes]stru
attr := s3response.ObjectAttributes(a)
if !attr.IsValid() {
debuglogger.Logf("invalid object attribute: %v\n", attr)
err = s3err.GetAPIError(s3err.ErrInvalidObjectAttributes)
err = s3err.GetInvalidArgumentErr(s3err.InvalidArgObjectAttributes, string(value))
break
}
attrs[attr] = struct{}{}
@@ -458,9 +445,13 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
objLockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode")
objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date")
if (objLockDate != "" && objLockModeHdr == "") || (objLockDate == "" && objLockModeHdr != "") {
debuglogger.Logf("one of 2 required params is missing: (lock date): %v, (lock mode): %v\n", objLockDate, objLockModeHdr)
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)
if objLockDate != "" && objLockModeHdr == "" {
debuglogger.Logf("the missing x-amz-object-lock-mode is required with x-amz-object-lock-retain-until-date")
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingObjectLockMode, "")
}
if objLockDate == "" && objLockModeHdr != "" {
debuglogger.Logf("the missing x-amz-object-lock-retain-until-date is required with x-amz-object-lock-mode")
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingObjectLockRetainDate, "")
}
var retainUntilDate time.Time
@@ -468,11 +459,11 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
rDate, err := time.Parse(time.RFC3339, objLockDate)
if err != nil {
debuglogger.Logf("failed to parse retain until date: %v\n", err)
return nil, s3err.GetAPIError(s3err.ErrInvalidRetainUntilDate)
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgRetainUntilDate, objLockDate)
}
if rDate.Before(time.Now()) {
debuglogger.Logf("expired retain until date: %v\n", rDate.Format(time.RFC3339))
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgPastObjectLockRetainDate, objLockDate)
}
retainUntilDate = rDate
}
@@ -483,14 +474,14 @@ func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
objLockMode != types.ObjectLockModeCompliance &&
objLockMode != types.ObjectLockModeGovernance {
debuglogger.Logf("invalid object lock mode: %v\n", objLockMode)
return nil, s3err.GetAPIError(s3err.ErrInvalidObjectLockMode)
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgObjectLockMode, objLockModeHdr)
}
legalHold := types.ObjectLockLegalHoldStatus(legalHoldHdr)
if legalHold != "" && legalHold != types.ObjectLockLegalHoldStatusOff && legalHold != types.ObjectLockLegalHoldStatusOn {
debuglogger.Logf("invalid object lock legal hold status: %v\n", legalHold)
return nil, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus)
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgLegalHoldStatus, legalHoldHdr)
}
return &objLockCfg{
@@ -928,25 +919,25 @@ func ParseTagging(data []byte, limit TagLimit) (map[string]string, error) {
// validate tag key length
if len(tag.Key) == 0 || len(tag.Key) > 128 {
debuglogger.Logf("tag key should 0 < tag.Key <= 128, key: %v", tag.Key)
return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
return nil, s3err.GetInvalidTagErr(s3err.ErrInvalidTagKey, tag.Key, "")
}
// validate tag key string chars
if !tagRule.MatchString(tag.Key) {
debuglogger.Logf("invalid tag key: %s", tag.Key)
return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
return nil, s3err.GetInvalidTagErr(s3err.ErrInvalidTagKey, tag.Key, "")
}
// validate tag value length
if len(tag.Value) > 256 {
debuglogger.Logf("invalid long tag value: (length): %v, (value): %v", len(tag.Value), tag.Value)
return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
return nil, s3err.GetInvalidTagErr(s3err.ErrInvalidTagValue, tag.Key, tag.Value)
}
// validate tag value string chars
if !tagRule.MatchString(tag.Value) {
debuglogger.Logf("invalid tag value: %s", tag.Value)
return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
return nil, s3err.GetInvalidTagErr(s3err.ErrInvalidTagValue, tag.Key, tag.Value)
}
// make sure there are no duplicate keys
@@ -1028,18 +1019,17 @@ func GetInt64(n *int64) int64 {
}
// ValidateCopySource parses and validates the copy-source
func ValidateCopySource(copysource string) error {
var err error
copysource, err = url.QueryUnescape(copysource)
func ValidateCopySource(input string) error {
copysource, err := url.QueryUnescape(input)
if err != nil {
debuglogger.Logf("invalid copy source encoding: %s", copysource)
return s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)
debuglogger.Logf("invalid copy source encoding: %s", input)
return s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, input)
}
bucket, rest, _ := strings.Cut(copysource, "/")
if !IsValidBucketName(bucket) {
debuglogger.Logf("invalid copy source bucket: %s", bucket)
return s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
return s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, input)
}
// cut till the versionId as it's the only query param
@@ -1051,7 +1041,7 @@ func ValidateCopySource(copysource string) error {
// in the gateway
if !IsObjectNameValid(object) {
debuglogger.Logf("invalid copy source object: %s", object)
return s3err.GetAPIError(s3err.ErrInvalidCopySourceObject)
return s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, object)
}
return nil
@@ -1132,3 +1122,21 @@ func NewTLSListener(network string, address string, getCertificateFunc func(*tls
}
return tls.NewListener(ln, config), nil
}
func DetectResourceType(ctx *fiber.Ctx) s3err.ResourceType {
path := ctx.Path()
if path == "" || path == "/" {
return s3err.ResourceTypeService
}
path = strings.TrimPrefix(path, "/")
_, rest, found := strings.Cut(path, "/")
if !found {
return s3err.ResourceTypeBucket
}
if rest == "" {
return s3err.ResourceTypeBucket
}
return s3err.ResourceTypeObject
}
s3api/utils/utils_test.go
+37 -37
View File
@@ -84,7 +84,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := createHttpRequestFromCtx(tt.args.ctx, tt.hdrs, 0, true)
got, err := createHttpRequestFromCtx(tt.args.ctx, tt.hdrs, 0)
if (err != nil) != tt.wantErr {
t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -169,7 +169,7 @@ func TestGetUserMetaData(t *testing.T) {
hdrs: [][2]string{
{"x-amz-meta-big", strings.Repeat("a", maxMetadataSize+1)},
},
wantErr: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
wantErr: s3err.GetMetadataTooLargeErr(2052, maxMetadataSize),
},
{
name: "metadata cumulative size exceeds limit (multiple headers)",
@@ -177,7 +177,7 @@ func TestGetUserMetaData(t *testing.T) {
{"x-amz-meta-a", strings.Repeat("a", maxMetadataSize/2)},
{"x-amz-meta-b", strings.Repeat("b", maxMetadataSize/2+10)},
},
wantErr: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
wantErr: s3err.GetMetadataTooLargeErr(2060, maxMetadataSize),
},
{
name: "duplicate keys combined",
@@ -398,7 +398,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypeMaxParts,
},
expected: expected{
err: s3err.GetInvalidMaxLimiterErr(string(LimiterTypeMaxParts)),
err: s3err.GetInvalidArgMaxLimiter(string(LimiterTypeMaxParts), "bla"),
res: 0,
},
},
@@ -409,7 +409,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypeMaxUploads,
},
expected: expected{
err: s3err.GetInvalidMaxLimiterErr(string(LimiterTypeMaxUploads)),
err: s3err.GetInvalidArgMaxLimiter(string(LimiterTypeMaxUploads), "invalid"),
res: 0,
},
},
@@ -420,7 +420,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypeMaxBuckets,
},
expected: expected{
err: s3err.GetInvalidMaxLimiterErr(string(LimiterTypeMaxBuckets)),
err: s3err.GetInvalidArgMaxLimiter(string(LimiterTypeMaxBuckets), "invalid"),
res: 0,
},
},
@@ -431,7 +431,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypeMaxKeys,
},
expected: expected{
err: s3err.GetInvalidMaxLimiterErr(string(LimiterTypeMaxKeys)),
err: s3err.GetInvalidArgMaxLimiter(string(LimiterTypeMaxKeys), "invalid"),
res: 0,
},
},
@@ -442,7 +442,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypeMaxKeys,
},
expected: expected{
err: s3err.GetNegativeMaxLimiterErr(string(LimiterTypeMaxKeys)),
err: s3err.GetInvalidArgNegativeMaxLimiter(string(LimiterTypeMaxKeys), "-5"),
res: 0,
},
},
@@ -453,7 +453,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypePartNumberMarker,
},
expected: expected{
err: s3err.GetNegativeMaxLimiterErr(string(LimiterTypePartNumberMarker)),
err: s3err.GetInvalidArgNegativeMaxLimiter(string(LimiterTypePartNumberMarker), "-5"),
res: 0,
},
},
@@ -464,7 +464,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypeMaxBuckets,
},
expected: expected{
err: s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "-12"),
res: 0,
},
},
@@ -475,7 +475,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypeVersionsMaxKeys,
},
expected: expected{
err: s3err.GetAPIError(s3err.ErrNegativeMaxKeys),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgNegativeMaxKeys, "-12"),
res: 0,
},
},
@@ -486,7 +486,7 @@ func TestParseMaxLimiter(t *testing.T) {
lt: LimiterTypeMaxBuckets,
},
expected: expected{
err: s3err.GetAPIError(s3err.ErrInvalidMaxBuckets),
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "25000"),
res: 0,
},
},
@@ -989,7 +989,7 @@ func TestExtractMetadataFromFields(t *testing.T) {
fields: map[string]string{
"x-amz-meta-big": strings.Repeat("a", maxMetadataSize-len("big")+1),
},
wantErr: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
wantErr: s3err.GetMetadataTooLargeErr(2049, maxMetadataSize),
},
}
@@ -1574,32 +1574,32 @@ func TestValidateCopySource(t *testing.T) {
err error
}{
// invalid encoding
{"invalid encoding 1", "%", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 2", "%2", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 3", "%G1", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 4", "%1Z", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 5", "%0H", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 6", "%XY", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 7", "%E", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 8", "hello%", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 9", "%%", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 10", "%2Gmore", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 11", "100%%sure", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 12", "%#00", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 13", "%0%0", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 14", "%?versionId=id", s3err.GetAPIError(s3err.ErrInvalidCopySourceEncoding)},
{"invalid encoding 1", "%", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%")},
{"invalid encoding 2", "%2", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%2")},
{"invalid encoding 3", "%G1", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%G1")},
{"invalid encoding 4", "%1Z", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%1Z")},
{"invalid encoding 5", "%0H", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%0H")},
{"invalid encoding 6", "%XY", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%XY")},
{"invalid encoding 7", "%E", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%E")},
{"invalid encoding 8", "hello%", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "hello%")},
{"invalid encoding 9", "%%", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%%")},
{"invalid encoding 10", "%2Gmore", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%2Gmore")},
{"invalid encoding 11", "100%%sure", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "100%%sure")},
{"invalid encoding 12", "%#00", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%#00")},
{"invalid encoding 13", "%0%0", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%0%0")},
{"invalid encoding 14", "%?versionId=id", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%?versionId=id")},
// invalid bucket name
{"invalid bucket name 1", "168.200.1.255/obj/foo", s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)},
{"invalid bucket name 2", "/0000:0db8:85a3:0000:0000:8a2e:0370:7224/smth", s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)},
{"invalid bucket name 3", "", s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)},
{"invalid bucket name 4", "//obj/foo", s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)},
{"invalid bucket name 5", "//obj/foo?versionId=id", s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)},
{"invalid bucket name 1", "168.200.1.255/obj/foo", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "168.200.1.255/obj/foo")},
{"invalid bucket name 2", "/0000:0db8:85a3:0000:0000:8a2e:0370:7224/smth", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "/0000:0db8:85a3:0000:0000:8a2e:0370:7224/smth")},
{"invalid bucket name 3", "", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "")},
{"invalid bucket name 4", "//obj/foo", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "//obj/foo")},
{"invalid bucket name 5", "//obj/foo?versionId=id", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "//obj/foo?versionId=id")},
// invalid object name
{"invalid object name 1", "bucket/../foo", s3err.GetAPIError(s3err.ErrInvalidCopySourceObject)},
{"invalid object name 2", "bucket/", s3err.GetAPIError(s3err.ErrInvalidCopySourceObject)},
{"invalid object name 3", "bucket", s3err.GetAPIError(s3err.ErrInvalidCopySourceObject)},
{"invalid object name 4", "bucket/../foo/dir/../../../", s3err.GetAPIError(s3err.ErrInvalidCopySourceObject)},
{"invalid object name 5", "bucket/.?versionId=smth", s3err.GetAPIError(s3err.ErrInvalidCopySourceObject)},
{"invalid object name 1", "bucket/../foo", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, "../foo")},
{"invalid object name 2", "bucket/", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, "")},
{"invalid object name 3", "bucket", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, "")},
{"invalid object name 4", "bucket/../foo/dir/../../../", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, "../foo/dir/../../../")},
{"invalid object name 5", "bucket/.?versionId=smth", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, ".")},
// success
{"no error 1", "bucket/object", nil},
{"no error 2", "bucket/object/key", nil},