feat: add browser-based POST object upload support

Closes #1648
Fixes #1980
Fixes #1981

This PR implements browser-based POST object uploads for S3-compatible form uploads. It adds support for handling `multipart/form-data` object uploads submitted from browsers, including streaming multipart parsing so file content is not buffered in memory, POST policy decoding and evaluation, SigV4-based form authorization, and integration with the existing `PutObject` backend flow. The implementation covers the full browser POST upload path, including validation of required form fields, credential scope and request date checks, signature verification, metadata extraction from `x-amz-meta-*` fields, checksum field parsing, object tagging conversion from XML into the query-string format expected by `PutObject`, and browser-compatible success handling through `success_action_status` and `success_action_redirect`. It also wires the new flow into the router and metrics layer and adds POST-specific error handling and debug logging across policy parsing, multipart parsing, and POST authorization. AWS S3 also accepts the `redirect` form field alongside `success_action_redirect`, but since AWS has marked `redirect` as deprecated and is planning to remove it, this gateway intentionally does not support it.
This commit is contained in:
niksis02
2026-03-31 22:44:54 +04:00
parent 69b23fe139
commit 285d130a47
26 changed files with 3662 additions and 188 deletions
+503
View File
@@ -0,0 +1,503 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
)
// POSTPolicy is the parsed browser-based upload policy document.
type POSTPolicy struct {
expiration time.Time
conditions []postPolicyCondition
}
// PostPolicyEvalInput is all the data required to evaluate the policy.
// Fields should contain already-expanded form values.
type PostPolicyEvalInput struct {
Bucket string
Key string
ContentLength int64
Fields map[string]string
}
// postPolicyCondition is the internal contract shared by all supported
// POST policy condition forms.
type postPolicyCondition interface {
validate() error
match(PostPolicyEvalInput) error
coveredField() string
}
// rawPolicy mirrors the JSON structure of an incoming POST policy document.
type rawPolicy struct {
Expiration string `json:"expiration"`
Conditions []json.RawMessage `json:"conditions"`
}
// ParsePOSTPolicyBase64 decodes and validates a base64-encoded POST policy.
func ParsePOSTPolicyBase64(encoded string) (*POSTPolicy, error) {
raw, err := decodeBase64Policy(encoded)
if err != nil {
return nil, err
}
var rp rawPolicy
err = json.Unmarshal(raw, &rp)
if err != nil {
debuglogger.Logf("invalid POST policy JSON: %v", err)
return nil, s3err.InvalidPolicyDocument.InvalidJSON()
}
if strings.TrimSpace(rp.Expiration) == "" {
debuglogger.Logf("POST policy is missing expiration")
return nil, s3err.InvalidPolicyDocument.MissingExpiration()
}
if len(rp.Conditions) == 0 {
debuglogger.Logf("POST policy is missing conditions")
return nil, s3err.InvalidPolicyDocument.MissingConditions()
}
exp, err := parseExpiration(rp.Expiration)
if err != nil {
return nil, err
}
conds := make([]postPolicyCondition, 0, len(rp.Conditions))
for _, rawCond := range rp.Conditions {
parsed, err := parseCondition(rawCond)
if err != nil {
return nil, err
}
conds = append(conds, parsed)
}
p := &POSTPolicy{
expiration: exp,
conditions: conds,
}
err = p.validate()
if err != nil {
return nil, err
}
return p, nil
}
// validate checks the parsed policy for structural correctness.
func (p *POSTPolicy) validate() error {
now := time.Now().UTC()
if p.expiration.Before(now) {
debuglogger.Logf("POST policy expired at %s", p.expiration.Format(time.RFC3339))
return s3err.InvalidPolicyDocument.PolicyExpired()
}
for _, cond := range p.conditions {
if err := cond.validate(); err != nil {
return err
}
}
return nil
}
// Evaluate returns nil if the input satisfies the policy.
// Otherwise it returns a deny reason.
func (p *POSTPolicy) Evaluate(in PostPolicyEvalInput) error {
// Every submitted form field must be present in conditions, except:
// x-amz-signature, file, policy, and x-ignore-*.
for field := range in.Fields {
if isIgnoredCoverageField(field) {
continue
}
if !p.hasConditionForField(field) {
debuglogger.Logf("POST policy does not cover input field: %s", field)
return s3err.InvalidPolicyDocument.ExtraInputField(field)
}
}
for _, cond := range p.conditions {
if err := cond.match(in); err != nil {
return err
}
}
return nil
}
// exactCondition represents either an object-form condition or an "eq" array
// condition that requires a field to match a single value exactly.
type exactCondition struct {
field string
value string
rawCondition []byte
}
// condition returns the policy expression used in condition failure messages.
func (c exactCondition) condition() string {
if len(c.rawCondition) == 0 {
// the key/value condition case
return fmt.Sprintf(`["eq", "$%s", "%s"]`, c.field, c.value)
}
return string(c.rawCondition)
}
// validate ensures the exact-match condition references a field.
func (c exactCondition) validate() error {
if strings.TrimSpace(c.field) == "" {
debuglogger.Logf("empty field in POST policy 'eq' condition")
return s3err.InvalidPolicyDocument.ConditionFailed(c.condition())
}
return nil
}
// match checks whether the resolved field value matches the expected value.
func (c exactCondition) match(in PostPolicyEvalInput) error {
got, ok := lookupField(in, c.field)
if !ok {
debuglogger.Logf("missing POST policy field %q for condition %s", c.field, c.condition())
return s3err.InvalidPolicyDocument.ConditionFailed(c.condition())
}
if got != c.value {
debuglogger.Logf("POST policy exact match failed for field %q: got %q want %q", c.field, got, c.value)
return s3err.InvalidPolicyDocument.ConditionFailed(c.condition())
}
return nil
}
// coveredField reports which form field this condition authorizes.
func (c exactCondition) coveredField() string { return c.field }
// startsWithCondition represents a policy rule that constrains a field by
// prefix rather than by exact equality.
type startsWithCondition struct {
field string
prefix string
rawCondition []byte
}
// condition returns the original policy expression for failure reporting.
func (c startsWithCondition) condition() string {
return string(c.rawCondition)
}
// validate ensures the starts-with condition references a field.
func (c startsWithCondition) validate() error {
if strings.TrimSpace(c.field) == "" {
debuglogger.Logf("empty field in POST policy 'starts-with' condition")
return s3err.InvalidPolicyDocument.ConditionFailed(c.condition())
}
return nil
}
// match checks whether the resolved field value satisfies the required prefix.
func (c startsWithCondition) match(in PostPolicyEvalInput) error {
got, ok := lookupField(in, c.field)
if !ok {
debuglogger.Logf("missing POST policy field %q for condition %s", c.field, c.condition())
return s3err.InvalidPolicyDocument.ConditionFailed(c.condition())
}
if !startsWithMatch(c.field, got, c.prefix) {
debuglogger.Logf("POST policy starts-with failed for field %q: got %q prefix %q", c.field, got, c.prefix)
return s3err.InvalidPolicyDocument.ConditionFailed(c.condition())
}
return nil
}
// coveredField reports which form field this condition authorizes.
func (c startsWithCondition) coveredField() string { return c.field }
// contentLengthRangeCondition enforces the allowed size range of the uploaded
// object body.
type contentLengthRangeCondition struct {
min int64
max int64
}
// validate accepts any parsed range and leaves size enforcement to match.
func (c contentLengthRangeCondition) validate() error {
return nil
}
// match rejects uploads whose content length falls outside the allowed range.
func (c contentLengthRangeCondition) match(in PostPolicyEvalInput) error {
if in.ContentLength > c.max {
debuglogger.Logf("POST policy content length %d exceeds max %d", in.ContentLength, c.max)
return s3err.GetAPIError(s3err.ErrEntityTooLarge)
}
if in.ContentLength < c.min {
debuglogger.Logf("POST policy content length %d is smaller than min %d", in.ContentLength, c.min)
return s3err.GetAPIError(s3err.ErrEntityTooSmall)
}
return nil
}
// Content length constraints apply to the uploaded body as a whole rather than
// to a named form field, so they do not participate in field coverage checks.
func (c contentLengthRangeCondition) coveredField() string { return "" }
// hasConditionForField reports whether the policy covers the supplied form
// field name.
func (p *POSTPolicy) hasConditionForField(field string) bool {
for _, cond := range p.conditions {
if cond.coveredField() == field {
return true
}
}
return false
}
// lookupField resolves policy field references against the canonical request
// state and submitted form fields.
func lookupField(in PostPolicyEvalInput, field string) (string, bool) {
// bucket and key are validated from the resolved request state
switch field {
case "bucket":
return in.Bucket, true
case "key":
return in.Key, true
}
if in.Fields == nil {
return "", false
}
v, ok := in.Fields[field]
return v, ok
}
// isIgnoredCoverageField reports whether a submitted field is exempt from the
// POST policy's field coverage requirement.
func isIgnoredCoverageField(field string) bool {
return field == "file" ||
field == "policy" ||
field == "x-amz-signature" ||
strings.HasPrefix(field, "x-ignore-")
}
// startsWithMatch applies AWS's starts-with matching rules for POST policy
// evaluation.
func startsWithMatch(field, value, prefix string) bool {
// AWS special-case:
// For starts-with on Content-Type, a comma-separated value is interpreted
// as a list and every entry must satisfy the prefix.
if field == "content-type" && strings.Contains(value, ",") {
parts := strings.SplitSeq(value, ",")
for part := range parts {
if !strings.HasPrefix(strings.TrimSpace(part), prefix) {
return false
}
}
return true
}
return strings.HasPrefix(value, prefix)
}
// decodeBase64Policy accepts both padded and raw standard base64 encodings for
// POST policy documents.
func decodeBase64Policy(s string) ([]byte, error) {
s = strings.TrimSpace(s)
if s == "" {
debuglogger.Logf("empty POST policy")
return nil, s3err.InvalidPolicyDocument.EmptyPolicy()
}
if raw, err := base64.StdEncoding.DecodeString(s); err == nil {
return raw, nil
}
if raw, err := base64.RawStdEncoding.DecodeString(s); err == nil {
return raw, nil
}
debuglogger.Logf("invalid POST policy base64 encoding")
return nil, s3err.InvalidPolicyDocument.InvalidBase64Encoding()
}
// parseExpiration parses the policy expiration timestamp and normalizes it to
// UTC.
func parseExpiration(s string) (time.Time, error) {
for _, layout := range []string{time.RFC3339Nano, time.RFC3339} {
if t, err := time.Parse(layout, s); err == nil {
return t.UTC(), nil
}
}
debuglogger.Logf("invalid POST policy expiration: %s", s)
return time.Time{}, s3err.InvalidPolicyDocument.InvalidExpiration(s)
}
// parseCondition converts one raw JSON condition into its internal validation
// and matching form.
func parseCondition(raw json.RawMessage) (postPolicyCondition, error) {
// Object form: {"content-type":"application/xml"}
var obj map[string]any
if err := json.Unmarshal(raw, &obj); err == nil && obj != nil {
if len(obj) != 1 {
debuglogger.Logf("POST policy simple condition must have exactly one property: %s", string(raw))
return nil, s3err.InvalidPolicyDocument.OnePropSimpleCondition()
}
for field, value := range obj {
s, ok := value.(string)
if !ok {
debuglogger.Logf("POST policy simple condition value must be string: %s", string(raw))
return nil, s3err.InvalidPolicyDocument.InvalidSimpleCondition()
}
return exactCondition{
field: strings.ToLower(field),
value: s,
}, nil
}
}
// Array form:
// ["eq", "$acl", "public-read"]
// ["starts-with", "$key", "user/eric/"]
// ["content-length-range", 1, 10485760]
var arr []json.RawMessage
if err := json.Unmarshal(raw, &arr); err != nil {
debuglogger.Logf("invalid POST policy condition: %s", string(raw))
return nil, s3err.InvalidPolicyDocument.InvalidCondition()
}
if len(arr) == 0 {
debuglogger.Logf("POST policy condition missing operation identifier")
return nil, s3err.InvalidPolicyDocument.MissingConditionOperationIdentifier()
}
var op string
if err := json.Unmarshal(arr[0], &op); err != nil {
debuglogger.Logf("invalid POST policy condition operation: %s", string(raw))
return nil, s3err.InvalidPolicyDocument.InvalidJSON()
}
switch op {
case "eq", "starts-with":
if len(arr) != 3 {
debuglogger.Logf("POST policy %s condition has wrong number of arguments: %s", op, string(raw))
return nil, s3err.InvalidPolicyDocument.IncorrectConditionArgumentsNumber(op)
}
var fieldRef string
if err := json.Unmarshal(arr[1], &fieldRef); err != nil {
debuglogger.Logf("invalid POST policy field reference: %s", string(raw))
return nil, s3err.InvalidPolicyDocument.InvalidJSON()
}
value, err := rawScalarToString(arr[2])
if err != nil {
debuglogger.Logf("invalid POST policy scalar value: %s", string(raw))
return nil, s3err.InvalidPolicyDocument.InvalidJSON()
}
if !strings.HasPrefix(fieldRef, "$") || len(fieldRef) == 1 {
debuglogger.Logf("invalid POST policy field reference format: %s", fieldRef)
return nil, s3err.InvalidPolicyDocument.ConditionFailed(string(raw))
}
// Normalize field names so condition checks line up with the parsed form
// map regardless of how they were written in the policy document.
field := strings.ToLower(fieldRef[1:])
if op == "eq" {
return exactCondition{field: field, value: value, rawCondition: raw}, nil
}
return startsWithCondition{field: field, prefix: value, rawCondition: raw}, nil
case "content-length-range":
if len(arr) != 3 {
debuglogger.Logf("POST policy %s condition has wrong number of arguments: %s", op, string(raw))
return nil, s3err.InvalidPolicyDocument.IncorrectConditionArgumentsNumber(op)
}
min, err := parseJSONInt64(arr[1], raw)
if err != nil {
return nil, err
}
max, err := parseJSONInt64(arr[2], raw)
if err != nil {
return nil, err
}
return contentLengthRangeCondition{min: min, max: max}, nil
default:
debuglogger.Logf("unknown POST policy operation: %s", op)
return nil, s3err.InvalidPolicyDocument.UnknownConditionOperation(op)
}
}
// parseJSONInt64 parses an integer condition operand from either a JSON number
// or a quoted decimal string.
func parseJSONInt64(raw json.RawMessage, rawCondition json.RawMessage) (int64, error) {
// try parsing as JSON number
var num json.Number
dec := json.NewDecoder(bytes.NewReader(raw))
dec.UseNumber()
if err := dec.Decode(&num); err == nil {
// Ensure it's a valid int64 (reject floats, exponents, overflow)
if v, err := num.Int64(); err == nil {
return v, nil
}
debuglogger.Logf("invalid POST policy integer value: %s", string(raw))
return 0, s3err.InvalidPolicyDocument.InvalidJSON()
}
// AWS also accepts quoted integers here.
var s string
if err := json.Unmarshal(raw, &s); err == nil {
v, err := strconv.ParseInt(s, 10, 64)
if err != nil {
debuglogger.Logf("invalid POST policy quoted integer: %s", s)
return 0, s3err.InvalidPolicyDocument.ConditionFailed(string(rawCondition))
}
return v, nil
}
debuglogger.Logf("invalid POST policy integer JSON: %s", string(raw))
return 0, s3err.InvalidPolicyDocument.InvalidJSON()
}
// rawScalarToString converts a JSON scalar that is valid in policy conditions
// into its string representation.
func rawScalarToString(raw json.RawMessage) (string, error) {
var v any
if err := json.Unmarshal(raw, &v); err != nil {
return "", err
}
switch x := v.(type) {
case string:
return x, nil
case json.Number:
return x.String(), nil
default:
return "", errors.New("unsupported type")
}
}
+355
View File
@@ -0,0 +1,355 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/base64"
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/versity/versitygw/s3err"
)
func encodePolicyForTest(t *testing.T, expiration time.Time, conditions []any, rawBase64 bool) string {
t.Helper()
policy := map[string]any{
"expiration": expiration.UTC().Format(time.RFC3339),
"conditions": conditions,
}
b, err := json.Marshal(policy)
assert.NoError(t, err)
if rawBase64 {
return base64.RawStdEncoding.EncodeToString(b)
}
return base64.StdEncoding.EncodeToString(b)
}
func encodeRawPolicyJSONForTest(t *testing.T, rawJSON string) string {
t.Helper()
return base64.StdEncoding.EncodeToString([]byte(rawJSON))
}
func mustParsePolicyForTest(t *testing.T, encoded string) *POSTPolicy {
t.Helper()
p, err := ParsePOSTPolicyBase64(encoded)
assert.NoError(t, err)
return p
}
func TestParsePOSTPolicyBase64_Success_AndEvaluate(t *testing.T) {
encoded := encodePolicyForTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "photos"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"},
[]any{"content-length-range", 1, 10},
}, false)
p := mustParsePolicyForTest(t, encoded)
assert.NoError(t, p.Evaluate(PostPolicyEvalInput{
Bucket: "photos",
Key: "uploads/image.jpg",
ContentLength: 5,
Fields: map[string]string{
"x-amz-algorithm": "AWS4-HMAC-SHA256",
"file": "ignored",
"policy": encoded,
"x-amz-signature": "ignored",
"x-ignore-meta": "ignored",
},
}))
}
func TestParsePOSTPolicyBase64_AcceptsRawBase64(t *testing.T) {
encoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
map[string]string{"bucket": "photos"},
}, true)
_, err := ParsePOSTPolicyBase64(encoded)
assert.Equal(t, error(nil), err)
}
func TestParsePOSTPolicyBase64_ConcreteParseErrors(t *testing.T) {
floatRangeEncoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
[]any{"content-length-range", 1.5, 10},
}, false)
unknownOpEncoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
[]any{"contains", "$key", "uploads/"},
}, false)
badQuotedRangeEncoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
[]any{"content-length-range", "abc", 10},
}, false)
tests := []struct {
name string
encoded string
expected error
}{
{
name: "empty policy",
encoded: " ",
expected: s3err.InvalidPolicyDocument.EmptyPolicy(),
},
{
name: "invalid base64",
encoded: "%%%not-base64%%%",
expected: s3err.InvalidPolicyDocument.InvalidBase64Encoding(),
},
{
name: "invalid json",
encoded: encodeRawPolicyJSONForTest(t, `{"expiration":`),
expected: s3err.InvalidPolicyDocument.InvalidJSON(),
},
{
name: "missing expiration",
encoded: encodeRawPolicyJSONForTest(t, `{
"conditions":[{"bucket":"photos"}]
}`),
expected: s3err.InvalidPolicyDocument.MissingExpiration(),
},
{
name: "missing conditions",
encoded: encodeRawPolicyJSONForTest(t, `{
"expiration":"2100-01-01T00:00:00Z"
}`),
expected: s3err.InvalidPolicyDocument.MissingConditions(),
},
{
name: "invalid expiration format",
encoded: encodeRawPolicyJSONForTest(t, `{
"expiration":"not-a-time",
"conditions":[{"bucket":"photos"}]
}`),
expected: s3err.InvalidPolicyDocument.InvalidExpiration("not-a-time"),
},
{
name: "expired policy",
encoded: encodePolicyForTest(t, time.Now().Add(-5*time.Minute), []any{
map[string]string{"bucket": "photos"},
}, false),
expected: s3err.InvalidPolicyDocument.PolicyExpired(),
},
{
name: "unknown operation",
encoded: unknownOpEncoded,
expected: s3err.InvalidPolicyDocument.UnknownConditionOperation("contains"),
},
{
name: "array condition missing op identifier",
encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
[]any{},
}, false),
expected: s3err.InvalidPolicyDocument.MissingConditionOperationIdentifier(),
},
{
name: "eq wrong number of args",
encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
[]any{"eq", "$key"},
}, false),
expected: s3err.InvalidPolicyDocument.IncorrectConditionArgumentsNumber("eq"),
},
{
name: "content-length-range wrong number of args",
encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
[]any{"content-length-range", 1},
}, false),
expected: s3err.InvalidPolicyDocument.IncorrectConditionArgumentsNumber("content-length-range"),
},
{
name: "invalid simple condition value type",
encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
map[string]any{"bucket": 1},
}, false),
expected: s3err.InvalidPolicyDocument.InvalidSimpleCondition(),
},
{
name: "simple condition with multiple properties",
encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
map[string]string{"bucket": "photos", "acl": "private"},
}, false),
expected: s3err.InvalidPolicyDocument.OnePropSimpleCondition(),
},
{
name: "condition not object or list",
encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
123,
}, false),
expected: s3err.InvalidPolicyDocument.InvalidCondition(),
},
{
name: "content-length-range with float",
encoded: floatRangeEncoded,
expected: s3err.InvalidPolicyDocument.InvalidJSON(),
},
{
name: "content-length-range with invalid quoted int",
encoded: badQuotedRangeEncoded,
expected: s3err.InvalidPolicyDocument.ConditionFailed(`["content-length-range","abc",10]`),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := ParsePOSTPolicyBase64(tt.encoded)
assert.Equal(t, tt.expected, err)
})
}
}
func TestPOSTPolicyEvaluate_ConcretePolicyRejections(t *testing.T) {
encoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
map[string]string{"bucket": "photos"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"},
[]any{"content-length-range", 2, 4},
}, false)
p := mustParsePolicyForTest(t, encoded)
tests := []struct {
name string
input PostPolicyEvalInput
expected error
}{
{
name: "extra field",
input: PostPolicyEvalInput{
Bucket: "photos",
Key: "uploads/image.jpg",
ContentLength: 3,
Fields: map[string]string{
"x-amz-algorithm": "AWS4-HMAC-SHA256",
"acl": "private",
},
},
expected: s3err.InvalidPolicyDocument.ExtraInputField("acl"),
},
{
name: "bucket mismatch",
input: PostPolicyEvalInput{
Bucket: "other-bucket",
Key: "uploads/image.jpg",
ContentLength: 3,
Fields: map[string]string{
"x-amz-algorithm": "AWS4-HMAC-SHA256",
},
},
expected: s3err.InvalidPolicyDocument.ConditionFailed(`["eq", "$bucket", "photos"]`),
},
{
name: "key prefix mismatch",
input: PostPolicyEvalInput{
Bucket: "photos",
Key: "tmp/image.jpg",
ContentLength: 3,
Fields: map[string]string{
"x-amz-algorithm": "AWS4-HMAC-SHA256",
},
},
expected: s3err.InvalidPolicyDocument.ConditionFailed(`["starts-with","$key","uploads/"]`),
},
{
name: "missing required field",
input: PostPolicyEvalInput{
Bucket: "photos",
Key: "uploads/image.jpg",
ContentLength: 3,
Fields: map[string]string{},
},
expected: s3err.InvalidPolicyDocument.ConditionFailed(`["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"]`),
},
{
name: "content too large",
input: PostPolicyEvalInput{
Bucket: "photos",
Key: "uploads/image.jpg",
ContentLength: 10,
Fields: map[string]string{
"x-amz-algorithm": "AWS4-HMAC-SHA256",
},
},
expected: s3err.GetAPIError(s3err.ErrEntityTooLarge),
},
{
name: "content too small",
input: PostPolicyEvalInput{
Bucket: "photos",
Key: "uploads/image.jpg",
ContentLength: 1,
Fields: map[string]string{
"x-amz-algorithm": "AWS4-HMAC-SHA256",
},
},
expected: s3err.GetAPIError(s3err.ErrEntityTooSmall),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := p.Evaluate(tt.input)
assert.Equal(t, tt.expected, err)
})
}
}
func TestPOSTPolicyEvaluate_IgnoresAllowedCoverageFields(t *testing.T) {
encoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
map[string]string{"bucket": "photos"},
[]any{"starts-with", "$key", "uploads/"},
}, false)
p := mustParsePolicyForTest(t, encoded)
err := p.Evaluate(PostPolicyEvalInput{
Bucket: "photos",
Key: "uploads/image.jpg",
ContentLength: 3,
Fields: map[string]string{
"file": "ignored",
"policy": "ignored",
"x-amz-signature": "ignored",
"x-ignore-meta": "ignored",
},
})
assert.Equal(t, error(nil), err)
}
func TestPOSTPolicyEvaluate_ContentTypeStartsWithList_AllEntriesMustMatch(t *testing.T) {
encoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{
map[string]string{"bucket": "photos"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"starts-with", "$Content-Type", "image/"},
}, false)
p := mustParsePolicyForTest(t, encoded)
err := p.Evaluate(PostPolicyEvalInput{
Bucket: "photos",
Key: "uploads/image.jpg",
ContentLength: 3,
Fields: map[string]string{
"content-type": "image/png,text/plain",
},
})
assert.Equal(t, s3err.InvalidPolicyDocument.ConditionFailed(`["starts-with","$Content-Type","image/"]`), err)
}
+5
View File
@@ -117,6 +117,7 @@ var (
ActionDeleteBucketWebsite = "s3_DeleteBucketWebsite"
ActionGetBucketPolicyStatus = "s3_GetBucketPolicyStatus"
ActionGetBucketLocation = "s3_GetBucketLocation"
ActionPostObject = "s3_PostObject"
// Admin actions
ActionAdminCreateUser = "admin_CreateUser"
@@ -504,4 +505,8 @@ func init() {
Name: "GetBucketLocation",
Service: "s3",
}
ActionMap[ActionPostObject] = Action{
Name: "PostObject",
Service: "s3",
}
}
+206
View File
@@ -16,6 +16,8 @@ package controllers
import (
"encoding/xml"
"net/http"
"net/url"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -23,6 +25,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3event"
@@ -93,3 +96,206 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) {
},
}, err
}
func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) {
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx)
parsed := utils.ContextKeyObjectPostResult.Get(ctx).(middlewares.PostObjectResult)
bucket := ctx.Params("bucket")
contentType := parsed.Fields["content-type"]
contentEncoding := parsed.Fields["content-encoding"]
contentDisposition := parsed.Fields["content-disposition"]
contentLanguage := parsed.Fields["content-language"]
cacheControl := parsed.Fields["cache-control"]
expires := parsed.Fields["expires"]
key, ok := parsed.Fields["key"]
if !ok || key == "" {
debuglogger.Logf("missing object key")
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.PostAuth.MissingField("key")
}
err := auth.VerifyAccess(ctx.Context(), c.be,
auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionWrite,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Action: auth.PutObjectAction,
IsPublicRequest: IsBucketPublic,
DisableACL: c.disableACL,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
// parse POST policy — absent for anonymous uploads to public buckets
policyBase64 := parsed.Fields["policy"]
if policyBase64 != "" {
policy, err := auth.ParsePOSTPolicyBase64(policyBase64)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
// Evaluate post policy
err = policy.Evaluate(auth.PostPolicyEvalInput{
Bucket: bucket,
Key: key,
ContentLength: parsed.ContentLength,
Fields: parsed.Fields,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
}
// convert object tagging from raw XML to Query string
// to pass PutObject, which expects the tagging to be a query string
var tagging string
if taggingXML, ok := parsed.Fields["tagging"]; ok {
tagging, err = utils.ConvertTaggingXMLToQueryString([]byte(taggingXML))
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
}
// parse checksum headers
checksums, err := utils.ParseCalculatedChecksumFields(parsed.Fields)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
// extract metadata
metadata, err := utils.ExtractMetadataFromFields(parsed.Fields)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
res, err := c.be.PutObject(ctx.Context(), s3response.PutObjectInput{
Bucket: &bucket,
Key: &key,
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ContentDisposition: &contentDisposition,
ContentLanguage: &contentLanguage,
CacheControl: &cacheControl,
Expires: &expires,
Body: parsed.FileRdr,
ContentLength: &parsed.ContentLength,
Tagging: &tagging,
Metadata: metadata,
ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
if successActionRedirect, ok := parsed.Fields["success_action_redirect"]; ok {
u, err := url.Parse(successActionRedirect)
if err == nil {
q := u.Query()
q.Set("bucket", bucket)
q.Set("key", key)
q.Set("etag", res.ETag)
u.RawQuery = q.Encode()
redirectURI := u.String()
return &Response{
Headers: map[string]*string{
"Location": &redirectURI,
},
MetaOpts: &MetaOptions{
ContentLength: parsed.FileRdr.Length(),
BucketOwner: parsedAcl.Owner,
ObjectETag: &res.ETag,
ObjectSize: parsed.FileRdr.Length(),
EventName: s3event.EventObjectCreatedPost,
Status: http.StatusSeeOther,
},
}, nil
}
}
respStatus := http.StatusNoContent
var respBody any
location := utils.GenerateObjectLocation(ctx, c.virtualDomain, bucket, key)
if successStatus, ok := parsed.Fields["success_action_status"]; ok {
switch successStatus {
case "200":
respStatus = http.StatusOK
case "201":
respStatus = http.StatusCreated
respBody = &s3response.PostResponse{
Bucket: bucket,
Key: key,
ETag: res.ETag,
Location: location,
}
}
}
return &Response{
Headers: map[string]*string{
"Etag": &res.ETag,
"Location": &location,
"x-amz-checksum-crc32": res.ChecksumCRC32,
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-version-id": utils.GetStringPtr(res.VersionID),
},
Data: respBody,
MetaOpts: &MetaOptions{
ContentLength: parsed.FileRdr.Length(),
BucketOwner: parsedAcl.Owner,
ObjectETag: &res.ETag,
ObjectSize: parsed.FileRdr.Length(),
EventName: s3event.EventObjectCreatedPost,
Status: respStatus,
},
}, nil
}
+533
View File
@@ -16,12 +16,20 @@ package controllers
import (
"context"
"encoding/base64"
"encoding/json"
"encoding/xml"
"io"
"net/http"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3event"
@@ -163,3 +171,528 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
})
}
}
// mockMpFileReader wraps an io.Reader and satisfies utils.MpFileReader.
// It tracks the number of bytes delivered to callers so that Length returns
// the same value that finalFileReader.Length would return after a real upload.
type mockMpFileReader struct {
r io.Reader
bytesRead int64
}
func (m *mockMpFileReader) Read(p []byte) (int, error) {
n, err := m.r.Read(p)
m.bytesRead += int64(n)
return n, err
}
func (m *mockMpFileReader) Length() int64 { return m.bytesRead }
func newMockFileReader(content string) *mockMpFileReader {
return &mockMpFileReader{r: strings.NewReader(content)}
}
func TestS3ApiController_POSTObject(t *testing.T) {
encodePOSTPolicyForControllerTest := func(t *testing.T, expiration time.Time, conditions []any) string {
t.Helper()
policy := map[string]any{
"expiration": expiration.UTC().Format(time.RFC3339),
"conditions": conditions,
}
b, err := json.Marshal(policy)
assert.NoError(t, err)
return base64.StdEncoding.EncodeToString(b)
}
postObjectLocalsForTest := func(parsed middlewares.PostObjectResult) map[utils.ContextKey]any {
return map[utils.ContextKey]any{
utils.ContextKeyIsRoot: true,
utils.ContextKeyParsedAcl: auth.ACL{
Owner: "root",
},
utils.ContextKeyAccount: auth.Account{
Access: "root",
Role: auth.RoleAdmin,
},
utils.ContextKeyRegion: "us-east-1",
utils.ContextKeyObjectPostResult: parsed,
}
}
marshalObjectTaggingForControllerTest := func(t *testing.T, tags []s3response.Tag) string {
t.Helper()
data, err := xml.Marshal(s3response.Tagging{
TagSet: s3response.TagSet{
Tags: tags,
},
})
assert.NoError(t, err)
return string(data)
}
validTaggingXML := marshalObjectTaggingForControllerTest(t, []s3response.Tag{
{Key: "project", Value: "alpha team"},
})
baseFields := map[string]string{
"key": "uploads/photo.jpg",
"file": "ignored",
"x-amz-signature": "ignored",
}
basePolicy := encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
})
baseFields["policy"] = basePolicy
location := "http://example.com/bucket/uploads%2Fphoto.jpg"
anonFields := map[string]string{
"key": "uploads/anon.bin",
"file": "ignored",
}
tests := []struct {
name string
input testInput
output testOutput
}{
{
name: "missing key",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"policy": basePolicy,
"file": "ignored",
"x-amz-signature": "ignored",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.PostAuth.MissingField("key"),
},
},
{
name: "verify access fails",
input: testInput{
locals: map[utils.ContextKey]any{
utils.ContextKeyIsRoot: false,
utils.ContextKeyParsedAcl: auth.ACL{
Owner: "user",
},
utils.ContextKeyAccount: auth.Account{
Access: "user",
Role: auth.RoleUser,
},
utils.ContextKeyRegion: "us-east-1",
utils.ContextKeyObjectPostResult: middlewares.PostObjectResult{
Fields: map[string]string{
"key": "key",
},
},
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "user",
},
},
err: s3err.GetAPIError(s3err.ErrAccessDenied),
},
},
{
name: "invalid policy",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": "%%%not-base64%%%",
"file": "ignored",
"x-amz-signature": "ignored",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.InvalidPolicyDocument.InvalidBase64Encoding(),
},
},
{
name: "policy evaluation fails on extra field",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": basePolicy,
"file": "ignored",
"x-amz-signature": "ignored",
"unexpected": "value",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.InvalidPolicyDocument.ExtraInputField("unexpected"),
},
},
{
name: "invalid tagging xml",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$tagging", "invalid-xml"},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"tagging": "invalid-xml",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMalformedXML),
},
},
{
name: "invalid checksum fields",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$x-amz-checksum-crc32", "invalid_base64_string"},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"x-amz-checksum-crc32": "invalid_base64_string",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"),
},
},
{
name: "metadata too large",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"starts-with", "$x-amz-meta-big", ""},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"x-amz-meta-big": strings.Repeat("a", 2050),
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
},
},
{
name: "backend returns error",
input: testInput{
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: baseFields,
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
},
},
{
name: "successful redirect response",
input: testInput{
beRes: s3response.PutObjectOutput{
ETag: "etag-123",
VersionID: "vid-123",
},
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$success_action_redirect", "https://client.example/upload-complete"},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"success_action_redirect": "https://client.example/upload-complete",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
Headers: map[string]*string{
"Location": utils.GetStringPtr("https://client.example/upload-complete?bucket=bucket&etag=etag-123&key=uploads%2Fphoto.jpg"),
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
ContentLength: int64(len("payload")),
ObjectETag: utils.GetStringPtr("etag-123"),
ObjectSize: int64(len("payload")),
EventName: s3event.EventObjectCreatedPost,
Status: 303,
},
},
},
},
{
name: "successful created response",
input: testInput{
beRes: s3response.PutObjectOutput{
ETag: "etag-123",
VersionID: "vid-123",
ChecksumCRC32: utils.GetStringPtr("crc32-out"),
ChecksumCRC32C: utils.GetStringPtr("crc32c-out"),
ChecksumSHA1: utils.GetStringPtr("sha1-out"),
ChecksumSHA256: utils.GetStringPtr("sha256-out"),
ChecksumCRC64NVME: utils.GetStringPtr("crc64-out"),
ChecksumType: types.ChecksumTypeComposite,
},
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$success_action_status", "201"},
[]any{"eq", "$tagging", validTaggingXML},
[]any{"eq", "$x-amz-meta-owner", "alice"},
[]any{"eq", "$x-amz-checksum-crc32", "ww2FVQ=="},
[]any{"eq", "$cache-control", "max-age=60"},
[]any{"eq", "$content-type", "image/jpeg"},
[]any{"eq", "$content-disposition", "inline"},
[]any{"eq", "$content-encoding", "gzip"},
[]any{"eq", "$content-language", "en-US"},
[]any{"eq", "$expires", "Fri, 21 Mar 2026 00:00:00 GMT"},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"success_action_status": "201",
"tagging": validTaggingXML,
"x-amz-meta-owner": "alice",
"x-amz-checksum-crc32": "ww2FVQ==",
"cache-control": "max-age=60",
"content-type": "image/jpeg",
"content-disposition": "inline",
"content-encoding": "gzip",
"content-language": "en-US",
"expires": "Fri, 21 Mar 2026 00:00:00 GMT",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
Headers: map[string]*string{
"Etag": utils.GetStringPtr("etag-123"),
"Location": &location,
"x-amz-checksum-crc32": utils.GetStringPtr("crc32-out"),
"x-amz-checksum-crc32c": utils.GetStringPtr("crc32c-out"),
"x-amz-checksum-crc64nvme": utils.GetStringPtr("crc64-out"),
"x-amz-checksum-sha1": utils.GetStringPtr("sha1-out"),
"x-amz-checksum-sha256": utils.GetStringPtr("sha256-out"),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("vid-123"),
},
Data: &s3response.PostResponse{
Bucket: "bucket",
Key: "uploads/photo.jpg",
ETag: "etag-123",
Location: location,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
ContentLength: int64(len("payload")),
ObjectETag: utils.GetStringPtr("etag-123"),
ObjectSize: int64(len("payload")),
EventName: s3event.EventObjectCreatedPost,
Status: 201,
},
},
},
},
{
name: "anonymous upload succeeds without policy",
input: testInput{
beRes: s3response.PutObjectOutput{
ETag: "etag-anon",
},
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: anonFields,
FileRdr: newMockFileReader("anon-payload"),
ContentLength: int64(len("anon-payload")),
}),
},
output: testOutput{
response: &Response{
Headers: map[string]*string{
"Etag": utils.GetStringPtr("etag-anon"),
"Location": utils.GetStringPtr("http://example.com/bucket/uploads%2Fanon.bin"),
"x-amz-checksum-crc32": nil,
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-type": nil,
"x-amz-version-id": utils.GetStringPtr(""), // empty, not nil — controller always returns &res.VersionID
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
ContentLength: int64(len("anon-payload")),
ObjectETag: utils.GetStringPtr("etag-anon"),
ObjectSize: int64(len("anon-payload")),
EventName: s3event.EventObjectCreatedPost,
Status: http.StatusNoContent,
},
},
},
},
{
name: "anonymous upload with policy is evaluated",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/anon.bin",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
// key condition intentionally omitted -> ExtraInputField for "key"
}),
"file": "ignored",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.InvalidPolicyDocument.ExtraInputField("key"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
be := &BackendMock{
PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
if tt.input.beErr != nil {
return s3response.PutObjectOutput{}, tt.input.beErr
}
// Drain the body as a real backend would, so that FileRdr.Length()
// reflects the actual bytes written after PutObject returns.
body, err := io.ReadAll(putObjectInput.Body)
assert.NoError(t, err)
if tt.name == "anonymous upload succeeds without policy" {
assert.Equal(t, "uploads/anon.bin", *putObjectInput.Key)
assert.Equal(t, "anon-payload", string(body))
}
if tt.name == "successful created response" {
assert.Equal(t, "bucket", *putObjectInput.Bucket)
assert.Equal(t, "uploads/photo.jpg", *putObjectInput.Key)
assert.Equal(t, "image/jpeg", *putObjectInput.ContentType)
assert.Equal(t, "gzip", *putObjectInput.ContentEncoding)
assert.Equal(t, "inline", *putObjectInput.ContentDisposition)
assert.Equal(t, "en-US", *putObjectInput.ContentLanguage)
assert.Equal(t, "max-age=60", *putObjectInput.CacheControl)
assert.Equal(t, "Fri, 21 Mar 2026 00:00:00 GMT", *putObjectInput.Expires)
assert.Equal(t, int64(len("payload")), *putObjectInput.ContentLength)
assert.Equal(t, "project=alpha+team", *putObjectInput.Tagging)
assert.Equal(t, map[string]string{"owner": "alice"}, putObjectInput.Metadata)
assert.Equal(t, utils.GetStringPtr("ww2FVQ=="), putObjectInput.ChecksumCRC32)
assert.Equal(t, "payload", string(body))
}
return tt.input.beRes.(s3response.PutObjectOutput), nil
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
},
}
ctrl := S3ApiController{
be: be,
}
testController(
t,
ctrl.POSTObject,
tt.output.response,
tt.output.err,
ctxInputs{
locals: tt.input.locals,
},
)
})
}
}
+1 -1
View File
@@ -32,7 +32,7 @@ import (
)
func TestS3ApiController_PutBucketTagging(t *testing.T) {
validTaggingBody, err := xml.Marshal(s3response.TaggingInput{
validTaggingBody, err := xml.Marshal(s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{
{
+1 -1
View File
@@ -34,7 +34,7 @@ import (
func TestS3ApiController_PutObjectTagging(t *testing.T) {
validTaggingBody, err := xml.Marshal(
s3response.TaggingInput{
s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{
{
+187
View File
@@ -0,0 +1,187 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"bytes"
"mime"
"strconv"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
)
const (
formFieldPolicy = "policy"
formFieldAlgorithm = "x-amz-algorithm"
formFieldCredential = "x-amz-credential"
formFieldDate = "x-amz-date"
formFieldSignature = "x-amz-signature"
aws4HMACSHA256 = "AWS4-HMAC-SHA256"
)
type PostObjectResult struct {
ContentLength int64
// FileRdr streams the file payload. Length() reports the exact number of
// file-content bytes read after the backend has consumed the body.
FileRdr utils.MpFileReader
Fields map[string]string
}
func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
contentLengthStr := ctx.Get("Content-Length")
reqContentLength, err := strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
debuglogger.Logf("invalid POST object Content-Length %q: %v", contentLengthStr, err)
return s3err.GetAPIError(s3err.ErrInvalidRequest)
}
mediaType, params, err := mime.ParseMediaType(ctx.Get("Content-Type"))
if err != nil || mediaType != fiber.MIMEMultipartForm {
debuglogger.Logf("invalid POST object Content-Type %q: mediaType=%q err=%v", ctx.Get("Content-Type"), mediaType, err)
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
}
boundary := params["boundary"]
if boundary == "" {
debuglogger.Logf("missing multipart boundary in POST object request")
return s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
bodyRdr := ctx.Request().BodyStream()
if bodyRdr == nil {
bodyRdr = bytes.NewReader(ctx.Body())
}
mpParser, err := utils.NewMultipartParser(bodyRdr, boundary, reqContentLength)
if err != nil {
return err
}
result, err := mpParser.Parse()
if err != nil {
return err
}
fields := result.Fields
policyB64 := fields[formFieldPolicy]
algorithm := fields[formFieldAlgorithm]
credentialStr := fields[formFieldCredential]
amzDate := fields[formFieldDate]
signatureHex := fields[formFieldSignature]
// Determine if the request carries form-based credentials.
// A request is considered signed if ANY of the five auth fields is
// present; in that case ALL of them are required.
hasAnyAuthField := policyB64 != "" || algorithm != "" || credentialStr != "" || amzDate != "" || signatureHex != ""
if hasAnyAuthField {
// Signed POST Object — validate every required auth field.
for _, field := range []struct {
key string
value string
}{
{formFieldPolicy, policyB64},
{formFieldAlgorithm, algorithm},
{formFieldCredential, credentialStr},
{formFieldDate, amzDate},
{formFieldSignature, signatureHex},
} {
if field.value == "" {
debuglogger.Logf("missing required POST object field: %s", field.key)
return s3err.PostAuth.MissingField(field.key)
}
}
if algorithm != aws4HMACSHA256 {
debuglogger.Logf("unsupported POST object signing algorithm: %s", algorithm)
return s3err.GetAPIError(s3err.ErrOnlyAws4HmacSha256)
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, amzDate)
if err != nil {
debuglogger.Logf("invalid POST object x-amz-date %q: %v", amzDate, err)
return s3err.GetAPIError(s3err.ErrInvalidDateHeader)
}
// Validate the dates difference
// TODO: Seems s3 doesn't validate this
err = utils.ValidateDate(tdate)
if err != nil {
return err
}
creds, err := utils.ParseCredentials(credentialStr, s3err.PostAuth)
if err != nil {
return err
}
if region != creds.Region {
debuglogger.Logf("incorrect POST object credential region: got %q want %q", creds.Region, region)
return s3err.MalformedAuth.IncorrectRegion(region, creds.Region)
}
account, err := acct.getAccount(creds.Access)
if err == auth.ErrNoSuchUser {
debuglogger.Logf("POST object access key not found: %s", creds.Access)
return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)
}
if err != nil {
debuglogger.Logf("failed to resolve POST object account %q: %v", creds.Access, err)
return err
}
utils.ContextKeyAccount.Set(ctx, account)
utils.ContextKeyIsRoot.Set(ctx, account.Access == root.Access)
expectedSig, err := utils.SignPostPolicy(policyB64, creds.Date, region, account.Secret)
if err != nil {
return err
}
if expectedSig != signatureHex {
debuglogger.Logf("POST object signature mismatch: expected %s got %s", expectedSig, signatureHex)
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
}
// Mark this request as authenticated so that
// AuthorizePublicBucketAccess (running after this middleware)
// skips its anonymous-access check.
utils.ContextKeyAuthenticated.Set(ctx, true)
}
// else: anonymous POST Object — no credentials in form fields.
// AuthorizePublicBucketAccess will verify public bucket access next.
utils.ContextKeyObjectPostResult.Set(ctx,
PostObjectResult{
Fields: fields,
FileRdr: result.FileRdr,
ContentLength: result.ContentLength,
},
)
return nil
}
}
+292
View File
@@ -0,0 +1,292 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"strings"
"testing"
"time"
"github.com/gofiber/fiber/v2"
"github.com/stretchr/testify/assert"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
)
// chainHandlers mimics controllers.ProcessHandlers: it calls each handler in
// sequence and stops on the first error. AuthorizePostObject (like all
// versitygw middlewares) returns nil without calling c.Next(), so they must
// be chained explicitly rather than relying on fiber's c.Next() mechanism.
func chainHandlers(handlers ...fiber.Handler) fiber.Handler {
return func(c *fiber.Ctx) error {
for _, h := range handlers {
if err := h(c); err != nil {
return err
}
}
return nil
}
}
// postObjectTestApp creates a fiber app that chains AuthorizePostObject with
// the provided follow-up handler on POST /:bucket.
func postObjectTestApp(root RootUserConfig, region string, next fiber.Handler) *fiber.App {
app := fiber.New(fiber.Config{
ErrorHandler: func(c *fiber.Ctx, err error) error {
if apiErr, ok := err.(s3err.APIError); ok {
return c.Status(apiErr.HTTPStatusCode).SendString(apiErr.Code)
}
return c.Status(500).SendString(err.Error())
},
})
app.Post("/:bucket", chainHandlers(AuthorizePostObject(root, nil, region), next))
return app
}
// buildMultipartBody returns a multipart/form-data body and its boundary.
// The given fields are written as form fields; fileContent is written as the
// "file" part.
func buildMultipartBody(t *testing.T, fields map[string]string, fileContent string) ([]byte, string) {
t.Helper()
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
for key, value := range fields {
assert.NoError(t, w.WriteField(key, value))
}
fw, err := w.CreateFormFile("file", "upload.bin")
assert.NoError(t, err)
_, err = io.WriteString(fw, fileContent)
assert.NoError(t, err)
assert.NoError(t, w.Close())
return buf.Bytes(), w.Boundary()
}
// encodePOSTPolicy encodes a minimal valid policy expiring 15 minutes in the
// future with the supplied conditions.
func encodePOSTPolicy(t *testing.T, conditions []any) string {
t.Helper()
policy := map[string]any{
"expiration": time.Now().Add(15 * time.Minute).UTC().Format(time.RFC3339),
"conditions": conditions,
}
b, err := json.Marshal(policy)
assert.NoError(t, err)
return base64.StdEncoding.EncodeToString(b)
}
func makePostRequest(t *testing.T, body []byte, boundary string) *http.Request {
t.Helper()
req, err := http.NewRequest(http.MethodPost, "/mybucket", bytes.NewReader(body))
assert.NoError(t, err)
req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%s", boundary))
// Set both the header and the int64 field: app.Test() serialises the
// request via req.Write() which uses req.ContentLength; fasthttp then
// re-reads it as the Content-Length header.
req.ContentLength = int64(len(body))
return req
}
// TestAuthorizePostObject_AnonymousRequest verifies that a POST with no auth
// fields succeeds: PostObjectResult is populated and ContextKeyAuthenticated
// is NOT set.
func TestAuthorizePostObject_AnonymousRequest(t *testing.T) {
var (
gotResult PostObjectResult
gotAuthenticated bool
)
app := postObjectTestApp(
RootUserConfig{Access: "root", Secret: "rootsecret"},
"us-east-1",
func(c *fiber.Ctx) error {
gotResult = utils.ContextKeyObjectPostResult.Get(c).(PostObjectResult)
gotAuthenticated = utils.ContextKeyAuthenticated.IsSet(c)
return c.SendStatus(http.StatusOK)
},
)
body, boundary := buildMultipartBody(t, map[string]string{
"key": "uploads/photo.jpg",
}, "file-content")
resp, err := app.Test(makePostRequest(t, body, boundary))
assert.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "uploads/photo.jpg", gotResult.Fields["key"])
assert.False(t, gotAuthenticated, "anonymous request must not set ContextKeyAuthenticated")
}
// TestAuthorizePostObject_AnonymousRequest_SetsPostObjectResult verifies that
// ContentType and FileRdr are populated for an anonymous upload.
func TestAuthorizePostObject_AnonymousRequest_SetsPostObjectResult(t *testing.T) {
var gotResult PostObjectResult
app := postObjectTestApp(
RootUserConfig{Access: "root", Secret: "rootsecret"},
"us-east-1",
func(c *fiber.Ctx) error {
gotResult = utils.ContextKeyObjectPostResult.Get(c).(PostObjectResult)
return c.SendStatus(http.StatusOK)
},
)
body, boundary := buildMultipartBody(t, map[string]string{
"key": "uploads/hello.txt",
"Content-Type": "text/plain",
}, "hello world")
resp, err := app.Test(makePostRequest(t, body, boundary))
assert.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "uploads/hello.txt", gotResult.Fields["key"])
assert.NotNil(t, gotResult.FileRdr)
}
// TestAuthorizePostObject_SignedRequest verifies that a correctly signed POST
// succeeds and sets ContextKeyAuthenticated.
func TestAuthorizePostObject_SignedRequest(t *testing.T) {
const (
region = "us-east-1"
accessKey = "testaccess"
secretKey = "testsecret"
)
now := time.Now().UTC()
dateShort := now.Format("20060102")
dateLong := now.Format("20060102T150405Z")
credential := fmt.Sprintf("%s/%s/%s/s3/aws4_request", accessKey, dateShort, region)
policyB64 := encodePOSTPolicy(t, []any{
map[string]string{"bucket": "mybucket"},
[]any{"starts-with", "$key", "uploads/"},
})
sig, err := utils.SignPostPolicy(policyB64, dateShort, region, secretKey)
assert.NoError(t, err)
var gotAuthenticated bool
app := postObjectTestApp(
RootUserConfig{Access: accessKey, Secret: secretKey},
region,
func(c *fiber.Ctx) error {
gotAuthenticated = utils.ContextKeyAuthenticated.IsSet(c)
return c.SendStatus(http.StatusOK)
},
)
body, boundary := buildMultipartBody(t, map[string]string{
"key": "uploads/photo.jpg",
"policy": policyB64,
"x-amz-algorithm": "AWS4-HMAC-SHA256",
"x-amz-credential": credential,
"x-amz-date": dateLong,
"x-amz-signature": sig,
}, "file-content")
resp, err := app.Test(makePostRequest(t, body, boundary))
assert.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.True(t, gotAuthenticated, "signed request must set ContextKeyAuthenticated")
}
// TestAuthorizePostObject_SignedRequest_WrongSignature verifies that a signed
// POST with a bad signature is rejected.
func TestAuthorizePostObject_SignedRequest_WrongSignature(t *testing.T) {
const (
region = "us-east-1"
accessKey = "testaccess"
secretKey = "testsecret"
)
now := time.Now().UTC()
dateShort := now.Format("20060102")
dateLong := now.Format("20060102T150405Z")
credential := fmt.Sprintf("%s/%s/%s/s3/aws4_request", accessKey, dateShort, region)
policyB64 := encodePOSTPolicy(t, []any{
map[string]string{"bucket": "mybucket"},
})
app := postObjectTestApp(
RootUserConfig{Access: accessKey, Secret: secretKey},
region,
func(c *fiber.Ctx) error { return c.SendStatus(http.StatusOK) },
)
body, boundary := buildMultipartBody(t, map[string]string{
"key": "uploads/photo.jpg",
"policy": policyB64,
"x-amz-algorithm": "AWS4-HMAC-SHA256",
"x-amz-credential": credential,
"x-amz-date": dateLong,
"x-amz-signature": "baadsignature00000000000",
}, "file-content")
resp, err := app.Test(makePostRequest(t, body, boundary))
assert.NoError(t, err)
assert.Equal(t, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch).HTTPStatusCode, resp.StatusCode)
}
// TestAuthorizePostObject_PartialAuthFields_ReturnsError verifies that
// providing only some auth fields (e.g. x-amz-algorithm only) is rejected.
func TestAuthorizePostObject_PartialAuthFields_ReturnsError(t *testing.T) {
app := postObjectTestApp(
RootUserConfig{Access: "root", Secret: "rootsecret"},
"us-east-1",
func(c *fiber.Ctx) error { return c.SendStatus(http.StatusOK) },
)
// Only algorithm is provided — credential, date, policy, signature absent.
body, boundary := buildMultipartBody(t, map[string]string{
"key": "uploads/photo.jpg",
"x-amz-algorithm": "AWS4-HMAC-SHA256",
}, "file-content")
resp, err := app.Test(makePostRequest(t, body, boundary))
assert.NoError(t, err)
assert.NotEqual(t, http.StatusOK, resp.StatusCode)
}
// TestAuthorizePostObject_InvalidContentType_ReturnsError verifies that a
// non-multipart Content-Type is rejected.
func TestAuthorizePostObject_InvalidContentType_ReturnsError(t *testing.T) {
app := postObjectTestApp(
RootUserConfig{Access: "root", Secret: "rootsecret"},
"us-east-1",
func(c *fiber.Ctx) error { return c.SendStatus(http.StatusOK) },
)
req, err := http.NewRequest(http.MethodPost, "/mybucket", strings.NewReader("body"))
assert.NoError(t, err)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Length", "4")
resp, err := app.Test(req)
assert.NoError(t, err)
assert.Equal(t, s3err.GetAPIError(s3err.ErrPreconditionFailed).HTTPStatusCode, resp.StatusCode)
}
+1 -1
View File
@@ -45,7 +45,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
// otherwise the middleware will return the caucht error
utils.ContextKeyAuthenticated.Set(ctx, true)
authData, err := utils.ParsePresignedURIParts(ctx)
authData, err := utils.ParsePresignedURIParts(ctx, region)
if err != nil {
return err
}
+1 -1
View File
@@ -33,7 +33,7 @@ import (
func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPermission auth.Action, permission auth.Permission, region string, streamBody bool) fiber.Handler {
return func(ctx *fiber.Ctx) error {
// skip for authenticated requests
if utils.IsPresignedURLAuth(ctx) || ctx.Get("Authorization") != "" {
if utils.IsPresignedURLAuth(ctx) || ctx.Get("Authorization") != "" || utils.ContextKeyAuthenticated.IsSet(ctx) {
return nil
}
+12
View File
@@ -1092,6 +1092,18 @@ func (sa *S3ApiRouter) Init() {
middlewares.ParseAcl(sa.be),
))
bucketRouter.Post("",
controllers.ProcessHandlers(
ctrl.POSTObject,
metrics.ActionPostObject,
services,
middlewares.BucketObjectNameValidator(),
middlewares.AuthorizePostObject(sa.root, sa.iam, sa.region),
middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionPostObject, auth.PutObjectAction, auth.PermissionWrite, sa.region, false),
middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin),
middlewares.ParseAcl(sa.be),
))
// object HEAD operation is not allowed with copy source
objectRouter.Head("/",
middlewares.MatchHeader("X-Amz-Copy-Source"),
+63 -15
View File
@@ -15,6 +15,9 @@
package utils
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
@@ -226,23 +229,13 @@ func ParseAuthorization(authorization string) (AuthData, error) {
switch key {
case "Credential":
creds := strings.Split(value, "/")
if len(creds) != 5 {
return a, s3err.MalformedAuth.MalformedCredential()
}
if creds[3] != "s3" {
return a, s3err.MalformedAuth.IncorrectService(creds[3])
}
if creds[4] != "aws4_request" {
return a, s3err.MalformedAuth.InvalidTerminal(creds[4])
}
_, err := time.Parse(yyyymmdd, creds[1])
creds, err := ParseCredentials(value, s3err.MalformedAuth)
if err != nil {
return a, s3err.MalformedAuth.InvalidDateFormat(creds[1])
return a, err
}
access = creds[0]
date = creds[1]
region = creds[2]
access = creds.Access
date = creds.Date
region = creds.Region
case "SignedHeaders":
signedHeaders = value
case "Signature":
@@ -260,6 +253,41 @@ func ParseAuthorization(authorization string) (AuthData, error) {
}, nil
}
type CredentialsScope struct {
Access string
Date string
Region string
}
type CredsError interface {
MalformedCredential() s3err.APIError
IncorrectService(string) s3err.APIError
IncorrectTerminal(string) s3err.APIError
InvalidDateFormat(string) s3err.APIError
}
func ParseCredentials(input string, errHandler CredsError) (*CredentialsScope, error) {
creds := strings.Split(input, "/")
if len(creds) != 5 {
return nil, errHandler.MalformedCredential()
}
if creds[3] != "s3" {
return nil, errHandler.IncorrectService(creds[3])
}
if creds[4] != "aws4_request" {
return nil, errHandler.IncorrectTerminal(creds[4])
}
_, err := time.Parse(yyyymmdd, creds[1])
if err != nil {
return nil, errHandler.InvalidDateFormat(creds[1])
}
return &CredentialsScope{
Access: creds[0],
Date: creds[1],
Region: creds[2],
}, nil
}
func removeSpace(str string) string {
var b strings.Builder
b.Grow(len(str))
@@ -270,3 +298,23 @@ func removeSpace(str string) string {
}
return b.String()
}
func SignPostPolicy(base64Policy, yyyymmdd, region, secretKey string) (string, error) {
signingKey := deriveSigningKey(secretKey, yyyymmdd, region)
sig := hmacSHA256(signingKey, []byte(base64Policy))
return hex.EncodeToString(sig), nil
}
func deriveSigningKey(secretKey, yyyymmdd, region string) []byte {
kDate := hmacSHA256([]byte("AWS4"+secretKey), []byte(yyyymmdd))
kRegion := hmacSHA256(kDate, []byte(region))
kService := hmacSHA256(kRegion, []byte(service))
kSigning := hmacSHA256(kService, []byte("aws4_request"))
return kSigning
}
func hmacSHA256(key, data []byte) []byte {
h := hmac.New(sha256.New, key)
h.Write(data)
return h.Sum(nil)
}
+14 -13
View File
@@ -24,19 +24,20 @@ import (
type ContextKey string
const (
ContextKeyRegion ContextKey = "region"
ContextKeyStartTime ContextKey = "start-time"
ContextKeyIsRoot ContextKey = "is-root"
ContextKeyRootAccessKey ContextKey = "root-access-key"
ContextKeyAccount ContextKey = "account"
ContextKeyAuthenticated ContextKey = "authenticated"
ContextKeyPublicBucket ContextKey = "public-bucket"
ContextKeyParsedAcl ContextKey = "parsed-acl"
ContextKeySkipResBodyLog ContextKey = "skip-res-body-log"
ContextKeyBodyReader ContextKey = "body-reader"
ContextKeySkip ContextKey = "__skip"
ContextKeyStack ContextKey = "stack"
ContextKeyBucketOwner ContextKey = "bucket-owner"
ContextKeyRegion ContextKey = "region"
ContextKeyStartTime ContextKey = "start-time"
ContextKeyIsRoot ContextKey = "is-root"
ContextKeyRootAccessKey ContextKey = "root-access-key"
ContextKeyAccount ContextKey = "account"
ContextKeyAuthenticated ContextKey = "authenticated"
ContextKeyPublicBucket ContextKey = "public-bucket"
ContextKeyParsedAcl ContextKey = "parsed-acl"
ContextKeySkipResBodyLog ContextKey = "skip-res-body-log"
ContextKeyBodyReader ContextKey = "body-reader"
ContextKeySkip ContextKey = "__skip"
ContextKeyStack ContextKey = "stack"
ContextKeyBucketOwner ContextKey = "bucket-owner"
ContextKeyObjectPostResult ContextKey = "object-post-result"
)
func (ck ContextKey) Values() []ContextKey {
+349
View File
@@ -0,0 +1,349 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"bufio"
"bytes"
"fmt"
"io"
"mime"
"net/textproto"
"strings"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
)
// MultipartParser parses S3 browser-based POST multipart/form-data in a streaming way.
// It buffers regular form fields, but it does not buffer the file part.
type MultipartParser struct {
br *bufio.Reader
boundary string
requestContentLength int64
bytesRead int64
}
// NewMultipartParser creates a new streaming multipart parser.
// boundary should be the raw boundary value from Content-Type, without the leading "--".
// If accidentally "--<boundary>" has been passed, it is normalized.
func NewMultipartParser(body io.Reader, boundary string, requestContentLength int64) (*MultipartParser, error) {
if body == nil {
debuglogger.Logf("multipart parser requires non-nil body reader")
return nil, fmt.Errorf("nil body reader")
}
if requestContentLength < 0 {
debuglogger.Logf("invalid multipart request content-length: %d", requestContentLength)
return nil, fmt.Errorf("invalid request content-length: %d", requestContentLength)
}
boundary = strings.TrimSpace(boundary)
boundary = strings.TrimPrefix(boundary, "--")
if boundary == "" {
debuglogger.Logf("multipart boundary is empty")
return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
return &MultipartParser{
br: bufio.NewReader(body),
boundary: boundary,
requestContentLength: requestContentLength,
}, nil
}
// MpFileReader is the streaming interface for the file part of a multipart POST.
// It extends io.Reader with a Length method that returns the number of file-content
// bytes actually delivered to callers (boundary and delimiter bytes are not counted).
type MpFileReader interface {
io.Reader
Length() int64
}
type MpParseResult struct {
// Fields contains all non-file form fields collected before the file part.
Fields map[string]string
// FileRdr streams the file payload without buffering the entire part in memory.
FileRdr MpFileReader
// ContentLength is the expected byte length of the file payload only.
ContentLength int64
}
// Parse parses all non-file fields and returns:
// - form values
// - a streaming file reader
// - file content length
//
// The returned file reader MUST be read until EOF, otherwise final-boundary
// validation is not triggered.
func (mp *MultipartParser) Parse() (*MpParseResult, error) {
fields := make(map[string]string)
if err := mp.expectInitialBoundary(); err != nil {
return nil, err
}
for {
headers, err := mp.readHeaders()
if err != nil {
return nil, err
}
cd, ok := headers[textproto.CanonicalMIMEHeaderKey("Content-Disposition")]
if !ok {
debuglogger.Logf("multipart part is missing Content-Disposition header")
return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
disp, params, err := mime.ParseMediaType(cd)
if err != nil {
debuglogger.Logf("invalid multipart Content-Disposition header %q: %v", cd, err)
return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
if disp != "form-data" {
debuglogger.Logf("unexpected multipart disposition: %s", disp)
return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
name := strings.ToLower(params["name"])
if name == "" {
debuglogger.Logf("multipart part is missing field name")
return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
_, hasFilename := params["filename"]
isFilePart := name == "file" || hasFilename
// At this point, headers + blank line have already been consumed,
// so bytesRead points exactly at the first byte of file content.
if isFilePart {
fileContentLength := mp.requestContentLength - mp.bytesRead - int64(len(mp.boundary)) - 8
if fileContentLength < 0 {
debuglogger.Logf("calculated negative multipart file content-length: %d", fileContentLength)
return nil, fmt.Errorf("calculated negative file content-length: %d", fileContentLength)
}
fr := &finalFileReader{
r: mp.br,
trailer: []byte("\r\n--" + mp.boundary + "--\r\n"),
}
return &MpParseResult{
Fields: fields,
FileRdr: fr,
ContentLength: fileContentLength,
}, nil
}
value, err := mp.readFieldValue()
if err != nil {
return nil, err
}
if strings.HasPrefix(name, "x-amz-meta-") {
val, ok := fields[name]
if ok {
fields[name] = val + "," + value
continue
}
}
fields[name] = value
}
}
func (mp *MultipartParser) expectInitialBoundary() error {
line, _, err := mp.readLine()
if err != nil {
return s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
want := "--" + mp.boundary
if line != want {
debuglogger.Logf("unexpected initial multipart boundary: got %q want %q", line, want)
return s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
return nil
}
func (mp *MultipartParser) readHeaders() (map[string]string, error) {
headers := make(map[string]string)
for {
line, _, err := mp.readLine()
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
// Blank line terminates headers.
if line == "" {
return headers, nil
}
key, value, ok := strings.Cut(line, ":")
if !ok {
debuglogger.Logf("invalid multipart header line: %q", line)
return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
key = textproto.CanonicalMIMEHeaderKey(strings.TrimSpace(key))
value = strings.TrimSpace(value)
headers[key] = value
}
}
// readFieldValue reads a regular form field until the next boundary line.
// It keeps exact field bytes except for the final CRLF that belongs to the boundary separator.
func (mp *MultipartParser) readFieldValue() (string, error) {
boundaryLine := "--" + mp.boundary
finalBoundaryLine := boundaryLine + "--"
var buf bytes.Buffer
for {
line, raw, err := mp.readLine()
if err != nil {
return "", err
}
switch line {
case boundaryLine:
trimTrailingCRLF(&buf)
return buf.String(), nil
case finalBoundaryLine:
debuglogger.Logf("multipart POST ended before file part was found")
return "", s3err.GetAPIError(s3err.ErrPOSTFileRequired)
default:
buf.Write(raw)
}
}
}
// readLine reads one CRLF-terminated line, counts consumed bytes,
// returns the line without trailing CRLF, and also the raw bytes including CRLF.
func (mp *MultipartParser) readLine() (string, []byte, error) {
s, err := mp.br.ReadString('\n')
if err != nil {
debuglogger.Logf("failed to read multipart line: %v", err)
return "", nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
mp.bytesRead += int64(len(s))
if !strings.HasSuffix(s, "\r\n") {
debuglogger.Logf("multipart line is not CRLF-terminated: %q", s)
return "", nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
}
return strings.TrimSuffix(s, "\r\n"), []byte(s), nil
}
func trimTrailingCRLF(buf *bytes.Buffer) {
b := buf.Bytes()
if len(b) >= 2 && b[len(b)-2] == '\r' && b[len(b)-1] == '\n' {
buf.Truncate(len(b) - 2)
}
}
// finalFileReader streams file bytes until it reaches the final multipart boundary:
//
// \r\n--<boundary>--\r\n
//
// Any epilogue bytes after that final boundary are ignored.
type finalFileReader struct {
r *bufio.Reader
// trailer is the exact byte sequence that terminates the file part.
trailer []byte
// buf keeps unread bytes plus a trailer-sized lookbehind window so
// boundary bytes split across reads are not emitted as file content.
buf []byte
// bytesRead counts only the file-content bytes delivered to callers.
// Boundary and delimiter bytes are never included in this count.
bytesRead int64
done bool
failed error
eof bool
}
func (r *finalFileReader) Read(p []byte) (int, error) {
if r.failed != nil {
return 0, r.failed
}
if r.done {
return 0, io.EOF
}
if len(p) == 0 {
return 0, nil
}
for {
// If the final boundary is already in the buffer, return only file
// bytes before it and stop cleanly once the boundary starts at buf[0].
if idx := bytes.Index(r.buf, r.trailer); idx >= 0 {
if idx == 0 {
r.buf = nil
r.done = true
return 0, io.EOF
}
n := copy(p, r.buf[:idx])
r.buf = r.buf[n:]
r.bytesRead += int64(n)
return n, nil
}
// Bytes before this point cannot be part of a future trailer match, so
// they are safe to release to the caller.
safe := len(r.buf) - len(r.trailer) + 1
if safe > 0 {
n := copy(p, r.buf[:safe])
r.buf = r.buf[n:]
r.bytesRead += int64(n)
return n, nil
}
if r.eof {
// Reaching EOF without finding the expected closing boundary means
// the multipart body was truncated or malformed.
debuglogger.Logf("multipart file stream ended before final boundary %q", string(r.trailer))
r.failed = s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)
return 0, r.failed
}
chunk := make([]byte, 4096)
n, err := r.r.Read(chunk)
if n > 0 {
r.buf = append(r.buf, chunk[:n]...)
}
if err == io.EOF {
r.eof = true
continue
}
if err != nil {
debuglogger.Logf("failed to read multipart file data: %v", err)
r.failed = err
return 0, err
}
}
}
// Length returns the total number of file-content bytes delivered to callers so far.
// Multipart boundary and delimiter bytes are never counted.
func (r *finalFileReader) Length() int64 {
return r.bytesRead
}
+514
View File
@@ -0,0 +1,514 @@
package utils
import (
"bufio"
"errors"
"io"
"strings"
"testing"
"github.com/versity/versitygw/s3err"
)
type chunkedReader struct {
data []byte
steps []int
pos int
idx int
}
func (r *chunkedReader) Read(p []byte) (int, error) {
if r.pos >= len(r.data) {
return 0, io.EOF
}
n := len(p)
if r.idx < len(r.steps) && r.steps[r.idx] < n {
n = r.steps[r.idx]
}
remaining := len(r.data) - r.pos
if remaining < n {
n = remaining
}
copy(p, r.data[r.pos:r.pos+n])
r.pos += n
r.idx++
return n, nil
}
func newMultipartParserForTest(t *testing.T, body, boundary string) *MultipartParser {
t.Helper()
mp, err := NewMultipartParser(strings.NewReader(body), boundary, int64(len(body)))
if err != nil {
t.Fatalf("new multipart parser: %v", err)
}
return mp
}
func TestNewMultipartParserValidation(t *testing.T) {
t.Parallel()
tests := []struct {
name string
body io.Reader
boundary string
length int64
wantError error
}{
{
name: "nil body",
boundary: "abc",
length: 1,
wantError: errors.New("nil body reader"),
},
{
name: "negative content length",
body: strings.NewReader("x"),
boundary: "abc",
length: -1,
wantError: errors.New("invalid request content-length: -1"),
},
{
name: "empty boundary",
body: strings.NewReader("x"),
boundary: " ",
length: 1,
wantError: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest),
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
_, err := NewMultipartParser(tt.body, tt.boundary, tt.length)
if err == nil {
t.Fatal("expected error")
}
if err.Error() != tt.wantError.Error() {
t.Fatalf("unexpected error: got %v want %v", err, tt.wantError)
}
})
}
}
func TestMultipartParserParseSuccess(t *testing.T) {
t.Parallel()
body := strings.Join([]string{
"--abc\r\n",
"Content-Disposition: form-data; name=\"key\"\r\n",
"\r\n",
"uploads/photo.jpg\r\n",
"--abc\r\n",
"Content-Disposition: form-data; name=\"success_action_status\"\r\n",
"\r\n",
"201\r\n",
"--abc\r\n",
"Content-Disposition: form-data; name=\"x-amz-meta-color\"\r\n",
"\r\n",
"blue\r\n",
"--abc\r\n",
"Content-Disposition: form-data; name=\"x-amz-meta-color\"\r\n",
"\r\n",
"green\r\n",
"--abc\r\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"photo.jpg\"\r\n",
"Content-Type: image/jpeg\r\n",
"\r\n",
"file-body-123",
"\r\n--abc--\r\n",
}, "")
mp := newMultipartParserForTest(t, body, "--abc")
got, err := mp.Parse()
if err != nil {
t.Fatalf("parse: %v", err)
}
if got.Fields["key"] != "uploads/photo.jpg" {
t.Fatalf("unexpected key field: %q", got.Fields["key"])
}
if got.Fields["success_action_status"] != "201" {
t.Fatalf("unexpected status field: %q", got.Fields["success_action_status"])
}
if got.Fields["x-amz-meta-color"] != "blue,green" {
t.Fatalf("unexpected merged metadata field: %q", got.Fields["x-amz-meta-color"])
}
if got.ContentLength != int64(len("file-body-123")) {
t.Fatalf("unexpected file content-length: got %d want %d", got.ContentLength, len("file-body-123"))
}
fileData, err := io.ReadAll(got.FileRdr)
if err != nil {
t.Fatalf("read file: %v", err)
}
if string(fileData) != "file-body-123" {
t.Fatalf("unexpected file data: %q", fileData)
}
n, err := got.FileRdr.Read(make([]byte, 1))
if n != 0 || !errors.Is(err, io.EOF) {
t.Fatalf("expected EOF after file reader drained, got n=%d err=%v", n, err)
}
}
func TestMultipartParserParsePreservesMultilineFieldValue(t *testing.T) {
t.Parallel()
body := strings.Join([]string{
"--abc\r\n",
"Content-Disposition: form-data; name=\"policy\"\r\n",
"\r\n",
"line-one\r\n",
"line-two\r\n",
"--abc\r\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"note.txt\"\r\n",
"\r\n",
"payload",
"\r\n--abc--\r\n",
}, "")
mp := newMultipartParserForTest(t, body, "abc")
got, err := mp.Parse()
if err != nil {
t.Fatalf("parse: %v", err)
}
if got.Fields["policy"] != "line-one\r\nline-two" {
t.Fatalf("unexpected multiline field: %q", got.Fields["policy"])
}
}
func TestMultipartParserRecognizesFilenameOnlyFilePart(t *testing.T) {
t.Parallel()
body := strings.Join([]string{
"--abc\r\n",
"Content-Disposition: form-data; name=\"key\"\r\n",
"\r\n",
"value\r\n",
"--abc\r\n",
"Content-Disposition: form-data; name=\"upload\"; filename=\"blob.bin\"\r\n",
"\r\n",
"xyz",
"\r\n--abc--\r\n",
}, "")
mp := newMultipartParserForTest(t, body, "abc")
got, err := mp.Parse()
if err != nil {
t.Fatalf("parse: %v", err)
}
if got.ContentLength != 3 {
t.Fatalf("unexpected file content-length: got %d want 3", got.ContentLength)
}
fileData, err := io.ReadAll(got.FileRdr)
if err != nil {
t.Fatalf("read file: %v", err)
}
if string(fileData) != "xyz" {
t.Fatalf("unexpected file data: %q", fileData)
}
}
func TestMultipartParserParseErrors(t *testing.T) {
t.Parallel()
tests := []struct {
name string
body string
want error
}{
{
name: "invalid initial boundary",
body: strings.Join([]string{
"--wrong\r\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"x\"\r\n",
"\r\n",
"payload\r\n",
"--abc--\r\n",
}, ""),
want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest),
},
{
name: "missing content disposition header",
body: strings.Join([]string{
"--abc\r\n",
"Content-Type: text/plain\r\n",
"\r\n",
"value\r\n",
"--abc--\r\n",
}, ""),
want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest),
},
{
name: "invalid header format",
body: strings.Join([]string{
"--abc\r\n",
"Content-Disposition form-data; name=\"key\"\r\n",
"\r\n",
"value\r\n",
"--abc--\r\n",
}, ""),
want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest),
},
{
name: "invalid content disposition media type",
body: strings.Join([]string{
"--abc\r\n",
"Content-Disposition: attachment; name=\"key\"\r\n",
"\r\n",
"value\r\n",
"--abc--\r\n",
}, ""),
want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest),
},
{
name: "missing content disposition name",
body: strings.Join([]string{
"--abc\r\n",
"Content-Disposition: form-data\r\n",
"\r\n",
"value\r\n",
"--abc--\r\n",
}, ""),
want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest),
},
{
name: "missing file part",
body: strings.Join([]string{
"--abc\r\n",
"Content-Disposition: form-data; name=\"key\"\r\n",
"\r\n",
"value\r\n",
"--abc--\r\n",
}, ""),
want: s3err.GetAPIError(s3err.ErrPOSTFileRequired),
},
{
name: "line without crlf terminator",
body: strings.Join([]string{
"--abc\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"x\"\r\n",
"\r\n",
"payload\r\n",
"--abc--\r\n",
}, ""),
want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest),
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
mp := newMultipartParserForTest(t, tt.body, "abc")
_, err := mp.Parse()
if !errors.Is(err, tt.want) {
t.Fatalf("unexpected error: got %v want %v", err, tt.want)
}
})
}
}
func TestMultipartParserFileReaderRequiresFinalBoundary(t *testing.T) {
t.Parallel()
body := strings.Join([]string{
"--abc\r\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"x\"\r\n",
"\r\n",
"payload-without-closing-boundary",
}, "")
mp := newMultipartParserForTest(t, body, "abc")
got, err := mp.Parse()
if err != nil {
t.Fatalf("parse: %v", err)
}
_, err = io.ReadAll(got.FileRdr)
if !errors.Is(err, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)) {
t.Fatalf("expected malformed post request, got %v", err)
}
}
func TestFinalFileReaderStopsAtFinalBoundary(t *testing.T) {
t.Parallel()
r := &finalFileReader{
r: bufio.NewReader(strings.NewReader("hello world\r\n--abc--\r\ntrailing-data")),
trailer: []byte("\r\n--abc--\r\n"),
}
got, err := io.ReadAll(r)
if err != nil {
t.Fatalf("read all: %v", err)
}
if string(got) != "hello world" {
t.Fatalf("unexpected body: got %q", got)
}
n, err := r.Read(make([]byte, 8))
if n != 0 {
t.Fatalf("expected zero bytes after EOF, got %d", n)
}
if !errors.Is(err, io.EOF) {
t.Fatalf("expected EOF after final boundary, got %v", err)
}
}
func TestFinalFileReaderHandlesBoundarySplitAcrossReads(t *testing.T) {
t.Parallel()
src := &chunkedReader{
data: []byte("hello world\r\n--abc--\r\nignored"),
steps: []int{5, 3, 2, 1, 4, 2, 3, 10},
}
r := &finalFileReader{
r: bufio.NewReader(src),
trailer: []byte("\r\n--abc--\r\n"),
}
got, err := io.ReadAll(r)
if err != nil {
t.Fatalf("read all: %v", err)
}
if string(got) != "hello world" {
t.Fatalf("unexpected body: got %q", got)
}
}
func TestFinalFileReaderMissingFinalBoundary(t *testing.T) {
t.Parallel()
r := &finalFileReader{
r: bufio.NewReader(strings.NewReader("hello world")),
trailer: []byte("\r\n--abc--\r\n"),
}
_, err := io.ReadAll(r)
if !errors.Is(err, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)) {
t.Fatalf("expected malformed post request, got %v", err)
}
}
func TestFinalFileReaderLengthCountsOnlyFileBytes(t *testing.T) {
t.Parallel()
fileContent := "hello world"
// trailer bytes must not be included in Length()
src := fileContent + "\r\n--abc--\r\n"
r := &finalFileReader{
r: bufio.NewReader(strings.NewReader(src)),
trailer: []byte("\r\n--abc--\r\n"),
}
got, err := io.ReadAll(r)
if err != nil {
t.Fatalf("read all: %v", err)
}
if string(got) != fileContent {
t.Fatalf("unexpected body: got %q want %q", got, fileContent)
}
if r.Length() != int64(len(fileContent)) {
t.Fatalf("unexpected length: got %d want %d", r.Length(), len(fileContent))
}
}
func TestFinalFileReaderLengthZeroBeforeRead(t *testing.T) {
t.Parallel()
r := &finalFileReader{
r: bufio.NewReader(strings.NewReader("data\r\n--b--\r\n")),
trailer: []byte("\r\n--b--\r\n"),
}
if r.Length() != 0 {
t.Fatalf("expected length 0 before any reads, got %d", r.Length())
}
}
func TestFinalFileReaderLengthIncrementalReads(t *testing.T) {
t.Parallel()
fileContent := "abcdefgh"
src := &chunkedReader{
data: []byte(fileContent + "\r\n--abc--\r\n"),
steps: []int{3, 3, 2, 10},
}
r := &finalFileReader{
r: bufio.NewReader(src),
trailer: []byte("\r\n--abc--\r\n"),
}
var total int
buf := make([]byte, 4)
for {
n, err := r.Read(buf)
total += n
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("unexpected read error: %v", err)
}
}
if r.Length() != int64(len(fileContent)) {
t.Fatalf("length after incremental reads: got %d want %d", r.Length(), len(fileContent))
}
if int64(total) != r.Length() {
t.Fatalf("total bytes returned by Read (%d) does not match Length() (%d)", total, r.Length())
}
}
func TestMultipartParserFileReaderLength(t *testing.T) {
t.Parallel()
fileContent := "file-body-123"
body := strings.Join([]string{
"--abc\r\n",
"Content-Disposition: form-data; name=\"key\"\r\n",
"\r\n",
"mykey\r\n",
"--abc\r\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"f.bin\"\r\n",
"\r\n",
fileContent,
"\r\n--abc--\r\n",
}, "")
mp := newMultipartParserForTest(t, body, "abc")
got, err := mp.Parse()
if err != nil {
t.Fatalf("parse: %v", err)
}
if got.FileRdr.Length() != 0 {
t.Fatalf("expected Length 0 before reading, got %d", got.FileRdr.Length())
}
if _, err := io.ReadAll(got.FileRdr); err != nil {
t.Fatalf("read all: %v", err)
}
if got.FileRdr.Length() != int64(len(fileContent)) {
t.Fatalf("unexpected Length after read: got %d want %d", got.FileRdr.Length(), len(fileContent))
}
}
+9 -29
View File
@@ -133,7 +133,7 @@ func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, strea
// &X-Amz-Expires=86400
// &X-Amz-SignedHeaders=host
// &X-Amz-Signature=1e68ad45c1db540284a4a1eca3884c293ba1a0ff63ab9db9a15b5b29dfa02cd8
func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
func ParsePresignedURIParts(ctx *fiber.Ctx, region string) (AuthData, error) {
a := AuthData{}
// Get and verify algorithm query parameter
@@ -149,34 +149,14 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
return a, s3err.QueryAuthErrors.MissingRequiredParams()
}
creds := strings.Split(credsQuery, "/")
if len(creds) != 5 {
return a, s3err.QueryAuthErrors.MalformedCredential()
}
// validate the service
if creds[3] != "s3" {
return a, s3err.QueryAuthErrors.IncorrectService(creds[3])
}
// validate the terminal
if creds[4] != "aws4_request" {
return a, s3err.QueryAuthErrors.IncorrectTerminal(creds[4])
}
// validate the date
_, err = time.Parse(yyyymmdd, creds[1])
creds, err := ParseCredentials(credsQuery, s3err.QueryAuthErrors)
if err != nil {
return a, s3err.QueryAuthErrors.InvalidDateFormat(creds[1])
return a, err
}
region, ok := ContextKeyRegion.Get(ctx).(string)
if !ok {
region = ""
}
// validate the region
if creds[2] != region {
return a, s3err.QueryAuthErrors.IncorrectRegion(region, creds[2])
if creds.Region != region {
return a, s3err.QueryAuthErrors.IncorrectRegion(region, creds.Region)
}
// Parse and validate Date query param
@@ -190,8 +170,8 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
return a, s3err.QueryAuthErrors.InvalidXAmzDateFormat()
}
if date[:8] != creds[1] {
return a, s3err.QueryAuthErrors.DateMismatch(creds[1], date[:8])
if date[:8] != creds.Date {
return a, s3err.QueryAuthErrors.DateMismatch(creds.Date, date[:8])
}
signature := ctx.Query("X-Amz-Signature")
@@ -211,9 +191,9 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
}
a.Signature = signature
a.Access = creds[0]
a.Access = creds.Access
a.Algorithm = algo
a.Region = creds[2]
a.Region = creds.Region
a.SignedHeaders = signedHdrs
a.Date = date
+86 -1
View File
@@ -103,6 +103,28 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (map[string]string, error)
return metadata, nil
}
func ExtractMetadataFromFields(fields map[string]string) (map[string]string, error) {
metadata := make(map[string]string)
var metadataSize int
for key, value := range fields {
if !strings.HasPrefix(key, "x-amz-meta-") {
continue
}
trimmedKey := key[11:]
metadataSize += len(trimmedKey) + len(value)
if metadataSize > maxMetadataSize {
debuglogger.Logf("total meta headers size exceeded the maximum allowed: (size): %v, (max): %v", metadataSize, maxMetadataSize)
return nil, s3err.GetAPIError(s3err.ErrMetadataTooLarge)
}
metadata[trimmedKey] = value
}
return metadata, nil
}
func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64, streamBody bool) (*http.Request, error) {
req := ctx.Request()
var body io.Reader
@@ -554,6 +576,53 @@ func ParseCalculatedChecksumHeaders(ctx *fiber.Ctx) (ChecksumValues, error) {
return checksums, nil
}
// ParseCalculatedChecksumFields parses and validates object POST checksum fields
func ParseCalculatedChecksumFields(fields map[string]string) (ChecksumValues, error) {
checksums := ChecksumValues{}
var hdrErr error
// Parse and validate checksum headers
for key, value := range fields {
// only check the headers with 'X-Amz-Checksum-' prefix
if !strings.HasPrefix(key, "x-amz-checksum-") {
continue
}
// "x-amz-checksum-type" and "x-amz-checksum-algorithm" aren't considered
// as invalid values, even if the s3 action doesn't expect these headers
switch key {
case "x-amz-checksum-type", "x-amz-checksum-algorithm":
continue
}
algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(key, "x-amz-checksum-")))
err := IsChecksumAlgorithmValid(algo)
if err != nil {
debuglogger.Logf("invalid checksum field: %s\n", key)
hdrErr = s3err.GetAPIError(s3err.ErrInvalidChecksumHeader)
break
}
checksums[algo] = value
}
if hdrErr != nil {
return checksums, hdrErr
}
if len(checksums) > 1 {
debuglogger.Logf("multiple checksum headers provided: %v\n", checksums.Headers())
return checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
}
for al, val := range checksums {
if !IsValidChecksum(val, al) {
return checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al))))
}
}
return checksums, nil
}
// ParseCompleteMpChecksumHeaders parses and validates
// the 'CompleteMultipartUpload' x-amz-checksum-x headers
// by supporting both 'checksum' and 'checksum-<part_length>' formats
@@ -805,7 +874,7 @@ var tagRule = regexp.MustCompile(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`)
// Parses and validates tagging
func ParseTagging(data []byte, limit TagLimit) (map[string]string, error) {
var tagging s3response.TaggingInput
var tagging s3response.Tagging
err := xml.Unmarshal(data, &tagging)
if err != nil {
debuglogger.Logf("invalid taggging: %s", data)
@@ -864,6 +933,22 @@ func ParseTagging(data []byte, limit TagLimit) (map[string]string, error) {
return tagSet, nil
}
// ConvertTaggingXMLToQueryString parses object tagging XML and converts it into the
// x-amz-tagging query-string form expected by PutObject requests.
func ConvertTaggingXMLToQueryString(data []byte) (string, error) {
tags, err := ParseTagging(data, TagLimitObject)
if err != nil {
return "", err
}
values := url.Values{}
for key, value := range tags {
values.Set(key, value)
}
return values.Encode(), nil
}
// Returns the provided string pointer
func GetStringPtr(str string) *string {
if str == "" {
+228 -8
View File
@@ -21,6 +21,7 @@ import (
"errors"
"math/rand"
"net/http"
"net/url"
"reflect"
"strings"
"testing"
@@ -815,6 +816,142 @@ func TestIsValidChecksum(t *testing.T) {
}
}
func TestExtractMetadataFromFields(t *testing.T) {
tests := []struct {
name string
fields map[string]string
want map[string]string
wantErr error
}{
{
name: "extracts metadata only",
fields: map[string]string{
"x-amz-meta-owner": "alice",
"x-amz-meta-env": "prod",
"key": "uploads/report.pdf",
},
want: map[string]string{
"owner": "alice",
"env": "prod",
},
},
{
name: "returns empty map when no metadata fields exist",
fields: map[string]string{
"key": "uploads/report.pdf",
"acl": "private",
"file": "ignored",
},
want: map[string]string{},
},
{
name: "allows empty metadata value",
fields: map[string]string{
"x-amz-meta-owner": "",
},
want: map[string]string{
"owner": "",
},
},
{
name: "metadata too large",
fields: map[string]string{
"x-amz-meta-big": strings.Repeat("a", maxMetadataSize-len("big")+1),
},
wantErr: s3err.GetAPIError(s3err.ErrMetadataTooLarge),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ExtractMetadataFromFields(tt.fields)
if !errors.Is(err, tt.wantErr) {
t.Fatalf("expected error %v, got %v", tt.wantErr, err)
}
if err != nil {
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Fatalf("unexpected metadata: got %v want %v", got, tt.want)
}
})
}
}
func TestParseCalculatedChecksumFields(t *testing.T) {
tests := []struct {
name string
fields map[string]string
want ChecksumValues
wantErr error
}{
{
name: "empty fields",
fields: map[string]string{
"key": "uploads/file.txt",
},
want: ChecksumValues{},
},
{
name: "single valid checksum field",
fields: map[string]string{
"x-amz-checksum-crc32": "ww2FVQ==",
},
want: ChecksumValues{
types.ChecksumAlgorithmCrc32: "ww2FVQ==",
},
},
{
name: "ignores algorithm and type helper fields",
fields: map[string]string{
"x-amz-checksum-algorithm": "CRC32",
"x-amz-checksum-type": "FULL_OBJECT",
"x-amz-checksum-crc32": "ww2FVQ==",
},
want: ChecksumValues{
types.ChecksumAlgorithmCrc32: "ww2FVQ==",
},
},
{
name: "invalid checksum field name",
fields: map[string]string{
"x-amz-checksum-madeup": "abc",
},
wantErr: s3err.GetAPIError(s3err.ErrInvalidChecksumHeader),
},
{
name: "multiple checksum fields",
fields: map[string]string{
"x-amz-checksum-crc32": "ww2FVQ==",
"x-amz-checksum-sha256": "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
},
wantErr: s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders),
},
{
name: "invalid checksum value",
fields: map[string]string{
"x-amz-checksum-crc32": "invalid_base64_string",
},
wantErr: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseCalculatedChecksumFields(tt.fields)
if !errors.Is(err, tt.wantErr) {
t.Fatalf("expected error %v, got %v", tt.wantErr, err)
}
if err != nil {
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Fatalf("unexpected checksums: got %v want %v", got, tt.want)
}
})
}
}
func TestIsChecksumTypeValid(t *testing.T) {
type args struct {
t types.ChecksumType
@@ -985,8 +1122,8 @@ func TestParseTagging(t *testing.T) {
}
return string(b)
}
getTagSet := func(lgth int) s3response.TaggingInput {
res := s3response.TaggingInput{
getTagSet := func(lgth int) s3response.Tagging {
res := s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{},
},
@@ -1002,7 +1139,7 @@ func TestParseTagging(t *testing.T) {
return res
}
type args struct {
data s3response.TaggingInput
data s3response.Tagging
overrideXML []byte
limit TagLimit
}
@@ -1015,7 +1152,7 @@ func TestParseTagging(t *testing.T) {
{
name: "valid tags within limit",
args: args{
data: s3response.TaggingInput{
data: s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{
{Key: "key1", Value: "value1"},
@@ -1037,6 +1174,24 @@ func TestParseTagging(t *testing.T) {
want: nil,
wantErr: s3err.GetAPIError(s3err.ErrMalformedXML),
},
{
name: "valid tags without namespace",
args: args{
overrideXML: []byte(`<?xml version="1.0"?><Tagging><TagSet><Tag><Key>key1</Key><Value>value1</Value></Tag><Tag><Key>key2</Key><Value>value2</Value></Tag></TagSet></Tagging>`),
limit: TagLimitObject,
},
want: map[string]string{"key1": "value1", "key2": "value2"},
wantErr: nil,
},
{
name: "valid tags with namespace",
args: args{
overrideXML: []byte(`<?xml version="1.0"?><Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><TagSet><Tag><Key>key1</Key><Value>value1</Value></Tag><Tag><Key>key2</Key><Value>value2</Value></Tag></TagSet></Tagging>`),
limit: TagLimitObject,
},
want: map[string]string{"key1": "value1", "key2": "value2"},
wantErr: nil,
},
{
name: "exceeds bucket tag limit",
args: args{
@@ -1058,7 +1213,7 @@ func TestParseTagging(t *testing.T) {
{
name: "invalid 0 length tag key",
args: args{
data: s3response.TaggingInput{
data: s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{{Key: "", Value: "value1"}},
},
@@ -1071,7 +1226,7 @@ func TestParseTagging(t *testing.T) {
{
name: "invalid long tag key",
args: args{
data: s3response.TaggingInput{
data: s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{{Key: genRandStr(130), Value: "value1"}},
},
@@ -1084,7 +1239,7 @@ func TestParseTagging(t *testing.T) {
{
name: "invalid long tag value",
args: args{
data: s3response.TaggingInput{
data: s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{{Key: "key", Value: genRandStr(257)}},
},
@@ -1097,7 +1252,7 @@ func TestParseTagging(t *testing.T) {
{
name: "duplicate tag key",
args: args{
data: s3response.TaggingInput{
data: s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{
{Key: "key", Value: "value1"},
@@ -1136,6 +1291,71 @@ func TestParseTagging(t *testing.T) {
}
}
func TestConvertTaggingXMLToQueryString(t *testing.T) {
tests := []struct {
name string
data s3response.Tagging
rawXML []byte
want string
wantErr error
}{
{
name: "success",
data: s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{
{Key: "project", Value: "versity gw"},
{Key: "team", Value: "storage"},
},
},
},
want: "project=versity+gw&team=storage",
},
{
name: "parse error",
rawXML: []byte("not xml"),
wantErr: s3err.GetAPIError(s3err.ErrMalformedXML),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var data []byte
if tt.rawXML != nil {
data = tt.rawXML
} else {
var err error
data, err = xml.Marshal(tt.data)
if err != nil {
t.Fatalf("error marshalling input: %v", err)
}
}
got, err := ConvertTaggingXMLToQueryString(data)
if !errors.Is(err, tt.wantErr) {
t.Fatalf("expected error %v, got %v", tt.wantErr, err)
}
if err != nil {
return
}
if got != tt.want {
t.Fatalf("unexpected tagging string: got %q want %q", got, tt.want)
}
values, err := url.ParseQuery(got)
if err != nil {
t.Fatalf("parse query: %v", err)
}
for _, tag := range tt.data.TagSet.Tags {
if values.Get(tag.Key) != tag.Value {
t.Fatalf("unexpected query value for %q: got %q want %q", tag.Key, values.Get(tag.Key), tag.Value)
}
}
})
}
}
func TestValidateCopySource(t *testing.T) {
tests := []struct {
name string
+147
View File
@@ -0,0 +1,147 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3err
import (
"fmt"
"net/http"
)
// Factory for building s3 object POST authentication errors.
func invalidPOSTObjectAuthErr(format string, args ...any) APIError {
return APIError{
Code: "InvalidArgument",
Description: fmt.Sprintf(format, args...),
HTTPStatusCode: http.StatusBadRequest,
}
}
type invalidPostAuthErr struct{}
func (invalidPostAuthErr) InvalidDateFormat(s string) APIError {
return invalidPOSTObjectAuthErr(
"incorrect date format %q. This date in the credential must be in the format \"yyyyMMdd\".",
s,
)
}
func (invalidPostAuthErr) MalformedCredential() APIError {
return invalidPOSTObjectAuthErr(
"the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".",
)
}
func (invalidPostAuthErr) IncorrectTerminal(s string) APIError {
return invalidPOSTObjectAuthErr("incorrect terminal %q. This endpoint uses \"aws4_request\".", s)
}
func (invalidPostAuthErr) IncorrectRegion(expected, actual string) APIError {
return invalidPOSTObjectAuthErr("the region %q is wrong; expecting %q", actual, expected)
}
func (invalidPostAuthErr) IncorrectService(s string) APIError {
return invalidPOSTObjectAuthErr("incorrect service %q. This endpoint belongs to \"s3\".", s)
}
func (invalidPostAuthErr) MissingField(field string) APIError {
return invalidPOSTObjectAuthErr("Bucket POST must contain a field named '%s'. If it is specified, please check the order of the fields.", field)
}
var PostAuth invalidPostAuthErr
// Factory for building s3 object POST authentication errors.
func invalidPolicyDocumentErr(format string, args ...any) APIError {
return APIError{
Code: "InvalidPolicyDocument",
Description: fmt.Sprintf(format, args...),
HTTPStatusCode: http.StatusBadRequest,
}
}
func invalidAccordingToPolicyErr(format string, args ...any) APIError {
return APIError{
Code: "AccessDenied",
Description: fmt.Sprintf(format, args...),
HTTPStatusCode: http.StatusForbidden,
}
}
type invalidPolicyDocument struct{}
func (invalidPolicyDocument) EmptyPolicy() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Expecting '{' but found End-of-Input")
}
func (invalidPolicyDocument) InvalidBase64Encoding() APIError {
return invalidPolicyDocumentErr("Invalid Policy: invalid Base64 encoding.")
}
func (invalidPolicyDocument) InvalidJSON() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid JSON.")
}
func (invalidPolicyDocument) MissingExpiration() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Policy missing expiration.")
}
func (invalidPolicyDocument) InvalidExpiration(exp string) APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid 'expiration' value: '%s'", exp)
}
func (invalidPolicyDocument) InvalidConditions() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid 'conditions' value: must be a List.")
}
func (invalidPolicyDocument) InvalidCondition() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid condition test: must be a List or Object.")
}
func (invalidPolicyDocument) MissingConditionOperationIdentifier() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid Condition: missing operation identifier.")
}
func (invalidPolicyDocument) UnknownConditionOperation(op string) APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid Condition: unknown operation '%s'.", op)
}
func (invalidPolicyDocument) IncorrectConditionArgumentsNumber(op string) APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid %s: wrong number of arguments.", op)
}
func (invalidPolicyDocument) MissingConditions() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Policy missing conditions.")
}
func (invalidPolicyDocument) OnePropSimpleCondition() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid Simple-Condition: Simple-Conditions must have exactly one property specified.")
}
func (invalidPolicyDocument) InvalidSimpleCondition() APIError {
return invalidPolicyDocumentErr("Invalid Policy: Invalid Simple-Condition: value must be a string.")
}
func (invalidPolicyDocument) ConditionFailed(condition string) APIError {
return invalidAccordingToPolicyErr("Invalid according to Policy: Policy Condition failed: %s", condition)
}
func (invalidPolicyDocument) ExtraInputField(field string) APIError {
return invalidAccordingToPolicyErr("Invalid according to Policy: Extra input fields: %s", field)
}
func (invalidPolicyDocument) PolicyExpired() APIError {
return invalidAccordingToPolicyErr("Invalid according to Policy: Policy expired.")
}
var InvalidPolicyDocument invalidPolicyDocument
+58 -63
View File
@@ -28,68 +28,63 @@ func authQueryParamError(format string, args ...any) APIError {
}
}
var QueryAuthErrors = struct {
UnsupportedAlgorithm func() APIError
MalformedCredential func() APIError
IncorrectService func(string) APIError
IncorrectRegion func(expected, actual string) APIError
IncorrectTerminal func(string) APIError
InvalidDateFormat func(string) APIError
DateMismatch func(expected, actual string) APIError
ExpiresTooLarge func() APIError
ExpiresNegative func() APIError
ExpiresNumber func() APIError
MissingRequiredParams func() APIError
InvalidXAmzDateFormat func() APIError
RequestNotYetValid func() APIError
RequestExpired func() APIError
InvalidAccessKeyId func() APIError
// a custom non-AWS error
OnlyHMACSupported func() APIError
SecurityTokenNotSupported func() APIError
}{
type queryAuthErrors struct{}
UnsupportedAlgorithm: func() APIError {
return authQueryParamError(`X-Amz-Algorithm only supports "AWS4-HMAC-SHA256 and AWS4-ECDSA-P256-SHA256"`)
},
MalformedCredential: func() APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting "<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request".`)
},
IncorrectService: func(s string) APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect service %q. This endpoint belongs to "s3".`, s)
},
IncorrectRegion: func(expected, actual string) APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; the region %q is wrong; expecting %q`, actual, expected)
},
IncorrectTerminal: func(s string) APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect terminal %q. This endpoint uses "aws4_request".`, s)
},
InvalidDateFormat: func(s string) APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect date format %q. This date in the credential must be in the format "yyyyMMdd".`, s)
},
DateMismatch: func(expected, actual string) APIError {
return authQueryParamError(`Invalid credential date %q. This date is not the same as X-Amz-Date: %q.`, expected, actual)
},
ExpiresTooLarge: func() APIError {
return authQueryParamError("X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds")
},
ExpiresNegative: func() APIError {
return authQueryParamError("X-Amz-Expires must be non-negative")
},
ExpiresNumber: func() APIError {
return authQueryParamError("X-Amz-Expires should be a number")
},
MissingRequiredParams: func() APIError {
return authQueryParamError("Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.")
},
InvalidXAmzDateFormat: func() APIError {
return authQueryParamError(`X-Amz-Date must be in the ISO8601 Long Format "yyyyMMdd'T'HHmmss'Z'"`)
},
// a custom non-AWS error
OnlyHMACSupported: func() APIError {
return authQueryParamError("X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\"")
},
SecurityTokenNotSupported: func() APIError {
return authQueryParamError("Authorization with X-Amz-Security-Token is not supported")
},
func (queryAuthErrors) UnsupportedAlgorithm() APIError {
return authQueryParamError(`X-Amz-Algorithm only supports "AWS4-HMAC-SHA256 and AWS4-ECDSA-P256-SHA256"`)
}
func (queryAuthErrors) MalformedCredential() APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting "<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request".`)
}
func (queryAuthErrors) IncorrectService(s string) APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect service %q. This endpoint belongs to "s3".`, s)
}
func (queryAuthErrors) IncorrectRegion(expected, actual string) APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; the region %q is wrong; expecting %q`, actual, expected)
}
func (queryAuthErrors) IncorrectTerminal(s string) APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect terminal %q. This endpoint uses "aws4_request".`, s)
}
func (queryAuthErrors) InvalidDateFormat(s string) APIError {
return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect date format %q. This date in the credential must be in the format "yyyyMMdd".`, s)
}
func (queryAuthErrors) DateMismatch(expected, actual string) APIError {
return authQueryParamError(`Invalid credential date %q. This date is not the same as X-Amz-Date: %q.`, expected, actual)
}
func (queryAuthErrors) ExpiresTooLarge() APIError {
return authQueryParamError("X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds")
}
func (queryAuthErrors) ExpiresNegative() APIError {
return authQueryParamError("X-Amz-Expires must be non-negative")
}
func (queryAuthErrors) ExpiresNumber() APIError {
return authQueryParamError("X-Amz-Expires should be a number")
}
func (queryAuthErrors) MissingRequiredParams() APIError {
return authQueryParamError("Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.")
}
func (queryAuthErrors) InvalidXAmzDateFormat() APIError {
return authQueryParamError(`X-Amz-Date must be in the ISO8601 Long Format "yyyyMMdd'T'HHmmss'Z'"`)
}
// a custom non-AWS error
func (queryAuthErrors) OnlyHMACSupported() APIError {
return authQueryParamError("X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\"")
}
func (queryAuthErrors) SecurityTokenNotSupported() APIError {
return authQueryParamError("Authorization with X-Amz-Security-Token is not supported")
}
var QueryAuthErrors queryAuthErrors
+13 -1
View File
@@ -186,6 +186,8 @@ const (
ErrInvalidChunkSize
ErrSlowDown
ErrMetadataTooLarge
ErrOnlyAws4HmacSha256
ErrInvalidDateHeader
// Non-AWS errors
ErrExistingObjectIsDirectory
@@ -439,7 +441,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrEntityTooSmall: {
Code: "EntityTooSmall",
Description: "Your proposed upload is smaller than the minimum allowed object size.",
Description: "Your proposed upload is smaller than the minimum allowed size",
HTTPStatusCode: http.StatusBadRequest,
},
ErrEntityTooLarge: {
@@ -843,6 +845,16 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Your metadata headers exceed the maximum allowed metadata size",
HTTPStatusCode: http.StatusBadRequest,
},
ErrOnlyAws4HmacSha256: {
Code: "InvalidArgument",
Description: "Only AWS4-HMAC-SHA256 is supported",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidDateHeader: {
Code: "InvalidArgument",
Description: "X-Amz-Date must be formated via ISO8601 Long format",
HTTPStatusCode: http.StatusBadRequest,
},
// non aws errors
ErrExistingObjectIsDirectory: {
+55 -46
View File
@@ -28,50 +28,59 @@ func malformedAuthError(format string, args ...any) APIError {
}
}
var MalformedAuth = struct {
InvalidDateFormat func(string) APIError
MalformedCredential func() APIError
MissingCredential func() APIError
MissingSignature func() APIError
MissingSignedHeaders func() APIError
InvalidTerminal func(string) APIError
IncorrectRegion func(expected, actual string) APIError
IncorrectService func(string) APIError
MalformedComponent func(string) APIError
MissingComponents func() APIError
DateMismatch func() APIError
}{
InvalidDateFormat: func(s string) APIError {
return malformedAuthError("incorrect date format %q. This date in the credential must be in the format \"yyyyMMdd\".", s)
},
MalformedCredential: func() APIError {
return malformedAuthError("the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".")
},
MissingCredential: func() APIError {
return malformedAuthError("missing Credential.")
},
MissingSignature: func() APIError {
return malformedAuthError("missing Signature.")
},
MissingSignedHeaders: func() APIError {
return malformedAuthError("missing SignedHeaders.")
},
InvalidTerminal: func(s string) APIError {
return malformedAuthError("incorrect terminal %q. This endpoint uses \"aws4_request\".", s)
},
IncorrectRegion: func(expected, actual string) APIError {
return malformedAuthError("the region %q is wrong; expecting %q", actual, expected)
},
IncorrectService: func(s string) APIError {
return malformedAuthError("incorrect service %q. This endpoint belongs to \"s3\".", s)
},
MalformedComponent: func(s string) APIError {
return malformedAuthError("the authorization component %q is malformed.", s)
},
MissingComponents: func() APIError {
return malformedAuthError("the authorization header requires three components: Credential, SignedHeaders, and Signature.")
},
DateMismatch: func() APIError {
return malformedAuthError("The authorization header is malformed; Invalid credential date. Date is not the same as X-Amz-Date.")
},
type malformedAuthErrors struct{}
func (malformedAuthErrors) InvalidDateFormat(s string) APIError {
return malformedAuthError(
"incorrect date format %q. This date in the credential must be in the format \"yyyyMMdd\".",
s,
)
}
func (malformedAuthErrors) MalformedCredential() APIError {
return malformedAuthError(
"the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".",
)
}
func (malformedAuthErrors) MissingCredential() APIError {
return malformedAuthError("missing Credential.")
}
func (malformedAuthErrors) MissingSignature() APIError {
return malformedAuthError("missing Signature.")
}
func (malformedAuthErrors) MissingSignedHeaders() APIError {
return malformedAuthError("missing SignedHeaders.")
}
func (malformedAuthErrors) IncorrectTerminal(s string) APIError {
return malformedAuthError("incorrect terminal %q. This endpoint uses \"aws4_request\".", s)
}
func (malformedAuthErrors) IncorrectRegion(expected, actual string) APIError {
return malformedAuthError("the region %q is wrong; expecting %q", actual, expected)
}
func (malformedAuthErrors) IncorrectService(s string) APIError {
return malformedAuthError("incorrect service %q. This endpoint belongs to \"s3\".", s)
}
func (malformedAuthErrors) MalformedComponent(s string) APIError {
return malformedAuthError("the authorization component %q is malformed.", s)
}
func (malformedAuthErrors) MissingComponents() APIError {
return malformedAuthError(
"the authorization header requires three components: Credential, SignedHeaders, and Signature.",
)
}
func (malformedAuthErrors) DateMismatch() APIError {
return malformedAuthError(
"The authorization header is malformed; Invalid credential date. Date is not the same as X-Amz-Date.",
)
}
var MalformedAuth malformedAuthErrors
+20 -2
View File
@@ -266,8 +266,18 @@ type Tagging struct {
TagSet TagSet `xml:"TagSet"`
}
type TaggingInput struct {
TagSet TagSet `xml:"TagSet"`
// UnmarshalXML accepts Tagging documents both with and without the S3 XML
// namespace, while xml.Marshal continues to emit the namespace via XMLName.
func (t *Tagging) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
type plain struct {
TagSet TagSet `xml:"TagSet"`
}
var p plain
if err := d.DecodeElement(&p, &start); err != nil {
return err
}
t.TagSet = p.TagSet
return nil
}
type DeleteObjects struct {
@@ -733,3 +743,11 @@ type CreateBucketConfiguration struct {
LocationConstraint *string
TagSet []types.Tag `xml:"Tags>Tag"`
}
type PostResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PostResponse"`
Location string
Bucket string
Key string
ETag string
}
+8 -5
View File
@@ -26,6 +26,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
func PutBucketTagging_non_existing_bucket(s *S3Conf) error {
@@ -196,11 +197,13 @@ func PutBucketTagging_success(s *S3Conf) error {
func PutBucketTagging_success_status(s *S3Conf) error {
testName := "PutBucketTagging_success_status"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
tagging := types.Tagging{
TagSet: []types.Tag{
{
Key: getPtr("key"),
Value: getPtr("val"),
tagging := s3response.Tagging{
TagSet: s3response.TagSet{
Tags: []s3response.Tag{
{
Key: "key",
Value: "value",
},
},
},
}
+1 -1
View File
@@ -215,7 +215,7 @@ func Authentication_credentials_invalid_terminal(s *S3Conf) error {
return err
}
return checkHTTPResponseApiErr(resp, s3err.MalformedAuth.InvalidTerminal("aws_request"))
return checkHTTPResponseApiErr(resp, s3err.MalformedAuth.IncorrectTerminal("aws_request"))
})
}