From 285d130a472274aca778709e0f7cc40a7a954d57 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Fri, 20 Mar 2026 17:48:59 +0400 Subject: [PATCH] feat: add browser-based POST object upload support Closes #1648 Fixes #1980 Fixes #1981 This PR implements browser-based POST object uploads for S3-compatible form uploads. It adds support for handling `multipart/form-data` object uploads submitted from browsers, including streaming multipart parsing so file content is not buffered in memory, POST policy decoding and evaluation, SigV4-based form authorization, and integration with the existing `PutObject` backend flow. The implementation covers the full browser POST upload path, including validation of required form fields, credential scope and request date checks, signature verification, metadata extraction from `x-amz-meta-*` fields, checksum field parsing, object tagging conversion from XML into the query-string format expected by `PutObject`, and browser-compatible success handling through `success_action_status` and `success_action_redirect`. It also wires the new flow into the router and metrics layer and adds POST-specific error handling and debug logging across policy parsing, multipart parsing, and POST authorization. AWS S3 also accepts the `redirect` form field alongside `success_action_redirect`, but since AWS has marked `redirect` as deprecated and is planning to remove it, this gateway intentionally does not support it. --- auth/post_policy.go | 503 +++++++++++++++++++ auth/post_policy_test.go | 355 ++++++++++++++ metrics/actions.go | 5 + s3api/controllers/bucket-post.go | 206 ++++++++ s3api/controllers/bucket-post_test.go | 533 +++++++++++++++++++++ s3api/controllers/bucket-put_test.go | 2 +- s3api/controllers/object-put_test.go | 2 +- s3api/middlewares/object-post-auth.go | 187 ++++++++ s3api/middlewares/object-post-auth_test.go | 292 +++++++++++ s3api/middlewares/presign-auth.go | 2 +- s3api/middlewares/public-bucket.go | 2 +- s3api/router.go | 12 + s3api/utils/auth-reader.go | 78 ++- s3api/utils/context-keys.go | 27 +- s3api/utils/multipart_reader.go | 349 ++++++++++++++ s3api/utils/multipart_reader_test.go | 514 ++++++++++++++++++++ s3api/utils/presign-auth-reader.go | 38 +- s3api/utils/utils.go | 87 +++- s3api/utils/utils_test.go | 236 ++++++++- s3err/post-object.go | 147 ++++++ s3err/presigned-urls.go | 121 +++-- s3err/s3err.go | 14 +- s3err/sigv4.go | 101 ++-- s3response/s3response.go | 22 +- tests/integration/PutBucketTagging.go | 13 +- tests/integration/sigv4_auth.go | 2 +- 26 files changed, 3662 insertions(+), 188 deletions(-) create mode 100644 auth/post_policy.go create mode 100644 auth/post_policy_test.go create mode 100644 s3api/middlewares/object-post-auth.go create mode 100644 s3api/middlewares/object-post-auth_test.go create mode 100644 s3api/utils/multipart_reader.go create mode 100644 s3api/utils/multipart_reader_test.go create mode 100644 s3err/post-object.go diff --git a/auth/post_policy.go b/auth/post_policy.go new file mode 100644 index 00000000..f04949d3 --- /dev/null +++ b/auth/post_policy.go @@ -0,0 +1,503 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3err" +) + +// POSTPolicy is the parsed browser-based upload policy document. +type POSTPolicy struct { + expiration time.Time + conditions []postPolicyCondition +} + +// PostPolicyEvalInput is all the data required to evaluate the policy. +// Fields should contain already-expanded form values. +type PostPolicyEvalInput struct { + Bucket string + Key string + ContentLength int64 + Fields map[string]string +} + +// postPolicyCondition is the internal contract shared by all supported +// POST policy condition forms. +type postPolicyCondition interface { + validate() error + match(PostPolicyEvalInput) error + coveredField() string +} + +// rawPolicy mirrors the JSON structure of an incoming POST policy document. +type rawPolicy struct { + Expiration string `json:"expiration"` + Conditions []json.RawMessage `json:"conditions"` +} + +// ParsePOSTPolicyBase64 decodes and validates a base64-encoded POST policy. +func ParsePOSTPolicyBase64(encoded string) (*POSTPolicy, error) { + raw, err := decodeBase64Policy(encoded) + if err != nil { + return nil, err + } + + var rp rawPolicy + err = json.Unmarshal(raw, &rp) + if err != nil { + debuglogger.Logf("invalid POST policy JSON: %v", err) + return nil, s3err.InvalidPolicyDocument.InvalidJSON() + } + + if strings.TrimSpace(rp.Expiration) == "" { + debuglogger.Logf("POST policy is missing expiration") + return nil, s3err.InvalidPolicyDocument.MissingExpiration() + } + if len(rp.Conditions) == 0 { + debuglogger.Logf("POST policy is missing conditions") + return nil, s3err.InvalidPolicyDocument.MissingConditions() + } + + exp, err := parseExpiration(rp.Expiration) + if err != nil { + return nil, err + } + + conds := make([]postPolicyCondition, 0, len(rp.Conditions)) + for _, rawCond := range rp.Conditions { + parsed, err := parseCondition(rawCond) + if err != nil { + return nil, err + } + conds = append(conds, parsed) + } + + p := &POSTPolicy{ + expiration: exp, + conditions: conds, + } + + err = p.validate() + if err != nil { + return nil, err + } + + return p, nil +} + +// validate checks the parsed policy for structural correctness. +func (p *POSTPolicy) validate() error { + now := time.Now().UTC() + if p.expiration.Before(now) { + debuglogger.Logf("POST policy expired at %s", p.expiration.Format(time.RFC3339)) + return s3err.InvalidPolicyDocument.PolicyExpired() + } + + for _, cond := range p.conditions { + if err := cond.validate(); err != nil { + return err + } + } + + return nil +} + +// Evaluate returns nil if the input satisfies the policy. +// Otherwise it returns a deny reason. +func (p *POSTPolicy) Evaluate(in PostPolicyEvalInput) error { + // Every submitted form field must be present in conditions, except: + // x-amz-signature, file, policy, and x-ignore-*. + for field := range in.Fields { + if isIgnoredCoverageField(field) { + continue + } + if !p.hasConditionForField(field) { + debuglogger.Logf("POST policy does not cover input field: %s", field) + return s3err.InvalidPolicyDocument.ExtraInputField(field) + } + } + + for _, cond := range p.conditions { + if err := cond.match(in); err != nil { + return err + } + } + + return nil +} + +// exactCondition represents either an object-form condition or an "eq" array +// condition that requires a field to match a single value exactly. +type exactCondition struct { + field string + value string + rawCondition []byte +} + +// condition returns the policy expression used in condition failure messages. +func (c exactCondition) condition() string { + if len(c.rawCondition) == 0 { + // the key/value condition case + return fmt.Sprintf(`["eq", "$%s", "%s"]`, c.field, c.value) + } + + return string(c.rawCondition) +} + +// validate ensures the exact-match condition references a field. +func (c exactCondition) validate() error { + if strings.TrimSpace(c.field) == "" { + debuglogger.Logf("empty field in POST policy 'eq' condition") + return s3err.InvalidPolicyDocument.ConditionFailed(c.condition()) + } + + return nil +} + +// match checks whether the resolved field value matches the expected value. +func (c exactCondition) match(in PostPolicyEvalInput) error { + got, ok := lookupField(in, c.field) + if !ok { + debuglogger.Logf("missing POST policy field %q for condition %s", c.field, c.condition()) + return s3err.InvalidPolicyDocument.ConditionFailed(c.condition()) + } + if got != c.value { + debuglogger.Logf("POST policy exact match failed for field %q: got %q want %q", c.field, got, c.value) + return s3err.InvalidPolicyDocument.ConditionFailed(c.condition()) + } + + return nil +} + +// coveredField reports which form field this condition authorizes. +func (c exactCondition) coveredField() string { return c.field } + +// startsWithCondition represents a policy rule that constrains a field by +// prefix rather than by exact equality. +type startsWithCondition struct { + field string + prefix string + rawCondition []byte +} + +// condition returns the original policy expression for failure reporting. +func (c startsWithCondition) condition() string { + return string(c.rawCondition) +} + +// validate ensures the starts-with condition references a field. +func (c startsWithCondition) validate() error { + if strings.TrimSpace(c.field) == "" { + debuglogger.Logf("empty field in POST policy 'starts-with' condition") + return s3err.InvalidPolicyDocument.ConditionFailed(c.condition()) + } + + return nil +} + +// match checks whether the resolved field value satisfies the required prefix. +func (c startsWithCondition) match(in PostPolicyEvalInput) error { + got, ok := lookupField(in, c.field) + if !ok { + debuglogger.Logf("missing POST policy field %q for condition %s", c.field, c.condition()) + return s3err.InvalidPolicyDocument.ConditionFailed(c.condition()) + } + if !startsWithMatch(c.field, got, c.prefix) { + debuglogger.Logf("POST policy starts-with failed for field %q: got %q prefix %q", c.field, got, c.prefix) + return s3err.InvalidPolicyDocument.ConditionFailed(c.condition()) + } + return nil +} + +// coveredField reports which form field this condition authorizes. +func (c startsWithCondition) coveredField() string { return c.field } + +// contentLengthRangeCondition enforces the allowed size range of the uploaded +// object body. +type contentLengthRangeCondition struct { + min int64 + max int64 +} + +// validate accepts any parsed range and leaves size enforcement to match. +func (c contentLengthRangeCondition) validate() error { + return nil +} + +// match rejects uploads whose content length falls outside the allowed range. +func (c contentLengthRangeCondition) match(in PostPolicyEvalInput) error { + if in.ContentLength > c.max { + debuglogger.Logf("POST policy content length %d exceeds max %d", in.ContentLength, c.max) + return s3err.GetAPIError(s3err.ErrEntityTooLarge) + } + if in.ContentLength < c.min { + debuglogger.Logf("POST policy content length %d is smaller than min %d", in.ContentLength, c.min) + return s3err.GetAPIError(s3err.ErrEntityTooSmall) + } + + return nil +} + +// Content length constraints apply to the uploaded body as a whole rather than +// to a named form field, so they do not participate in field coverage checks. +func (c contentLengthRangeCondition) coveredField() string { return "" } + +// hasConditionForField reports whether the policy covers the supplied form +// field name. +func (p *POSTPolicy) hasConditionForField(field string) bool { + for _, cond := range p.conditions { + if cond.coveredField() == field { + return true + } + } + + return false +} + +// lookupField resolves policy field references against the canonical request +// state and submitted form fields. +func lookupField(in PostPolicyEvalInput, field string) (string, bool) { + // bucket and key are validated from the resolved request state + switch field { + case "bucket": + return in.Bucket, true + case "key": + return in.Key, true + } + + if in.Fields == nil { + return "", false + } + + v, ok := in.Fields[field] + return v, ok +} + +// isIgnoredCoverageField reports whether a submitted field is exempt from the +// POST policy's field coverage requirement. +func isIgnoredCoverageField(field string) bool { + return field == "file" || + field == "policy" || + field == "x-amz-signature" || + strings.HasPrefix(field, "x-ignore-") +} + +// startsWithMatch applies AWS's starts-with matching rules for POST policy +// evaluation. +func startsWithMatch(field, value, prefix string) bool { + // AWS special-case: + // For starts-with on Content-Type, a comma-separated value is interpreted + // as a list and every entry must satisfy the prefix. + if field == "content-type" && strings.Contains(value, ",") { + parts := strings.SplitSeq(value, ",") + for part := range parts { + if !strings.HasPrefix(strings.TrimSpace(part), prefix) { + return false + } + } + return true + } + return strings.HasPrefix(value, prefix) +} + +// decodeBase64Policy accepts both padded and raw standard base64 encodings for +// POST policy documents. +func decodeBase64Policy(s string) ([]byte, error) { + s = strings.TrimSpace(s) + if s == "" { + debuglogger.Logf("empty POST policy") + return nil, s3err.InvalidPolicyDocument.EmptyPolicy() + } + + if raw, err := base64.StdEncoding.DecodeString(s); err == nil { + return raw, nil + } + if raw, err := base64.RawStdEncoding.DecodeString(s); err == nil { + return raw, nil + } + + debuglogger.Logf("invalid POST policy base64 encoding") + return nil, s3err.InvalidPolicyDocument.InvalidBase64Encoding() +} + +// parseExpiration parses the policy expiration timestamp and normalizes it to +// UTC. +func parseExpiration(s string) (time.Time, error) { + for _, layout := range []string{time.RFC3339Nano, time.RFC3339} { + if t, err := time.Parse(layout, s); err == nil { + return t.UTC(), nil + } + } + debuglogger.Logf("invalid POST policy expiration: %s", s) + return time.Time{}, s3err.InvalidPolicyDocument.InvalidExpiration(s) +} + +// parseCondition converts one raw JSON condition into its internal validation +// and matching form. +func parseCondition(raw json.RawMessage) (postPolicyCondition, error) { + // Object form: {"content-type":"application/xml"} + var obj map[string]any + if err := json.Unmarshal(raw, &obj); err == nil && obj != nil { + if len(obj) != 1 { + debuglogger.Logf("POST policy simple condition must have exactly one property: %s", string(raw)) + return nil, s3err.InvalidPolicyDocument.OnePropSimpleCondition() + } + + for field, value := range obj { + s, ok := value.(string) + if !ok { + debuglogger.Logf("POST policy simple condition value must be string: %s", string(raw)) + return nil, s3err.InvalidPolicyDocument.InvalidSimpleCondition() + } + + return exactCondition{ + field: strings.ToLower(field), + value: s, + }, nil + } + } + + // Array form: + // ["eq", "$acl", "public-read"] + // ["starts-with", "$key", "user/eric/"] + // ["content-length-range", 1, 10485760] + var arr []json.RawMessage + if err := json.Unmarshal(raw, &arr); err != nil { + debuglogger.Logf("invalid POST policy condition: %s", string(raw)) + return nil, s3err.InvalidPolicyDocument.InvalidCondition() + } + if len(arr) == 0 { + debuglogger.Logf("POST policy condition missing operation identifier") + return nil, s3err.InvalidPolicyDocument.MissingConditionOperationIdentifier() + } + + var op string + if err := json.Unmarshal(arr[0], &op); err != nil { + debuglogger.Logf("invalid POST policy condition operation: %s", string(raw)) + return nil, s3err.InvalidPolicyDocument.InvalidJSON() + } + + switch op { + case "eq", "starts-with": + if len(arr) != 3 { + debuglogger.Logf("POST policy %s condition has wrong number of arguments: %s", op, string(raw)) + return nil, s3err.InvalidPolicyDocument.IncorrectConditionArgumentsNumber(op) + } + + var fieldRef string + if err := json.Unmarshal(arr[1], &fieldRef); err != nil { + debuglogger.Logf("invalid POST policy field reference: %s", string(raw)) + return nil, s3err.InvalidPolicyDocument.InvalidJSON() + } + value, err := rawScalarToString(arr[2]) + if err != nil { + debuglogger.Logf("invalid POST policy scalar value: %s", string(raw)) + return nil, s3err.InvalidPolicyDocument.InvalidJSON() + } + + if !strings.HasPrefix(fieldRef, "$") || len(fieldRef) == 1 { + debuglogger.Logf("invalid POST policy field reference format: %s", fieldRef) + return nil, s3err.InvalidPolicyDocument.ConditionFailed(string(raw)) + } + + // Normalize field names so condition checks line up with the parsed form + // map regardless of how they were written in the policy document. + field := strings.ToLower(fieldRef[1:]) + if op == "eq" { + return exactCondition{field: field, value: value, rawCondition: raw}, nil + } + + return startsWithCondition{field: field, prefix: value, rawCondition: raw}, nil + case "content-length-range": + if len(arr) != 3 { + debuglogger.Logf("POST policy %s condition has wrong number of arguments: %s", op, string(raw)) + return nil, s3err.InvalidPolicyDocument.IncorrectConditionArgumentsNumber(op) + } + + min, err := parseJSONInt64(arr[1], raw) + if err != nil { + return nil, err + } + max, err := parseJSONInt64(arr[2], raw) + if err != nil { + return nil, err + } + + return contentLengthRangeCondition{min: min, max: max}, nil + default: + debuglogger.Logf("unknown POST policy operation: %s", op) + return nil, s3err.InvalidPolicyDocument.UnknownConditionOperation(op) + } +} + +// parseJSONInt64 parses an integer condition operand from either a JSON number +// or a quoted decimal string. +func parseJSONInt64(raw json.RawMessage, rawCondition json.RawMessage) (int64, error) { + // try parsing as JSON number + var num json.Number + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + + if err := dec.Decode(&num); err == nil { + // Ensure it's a valid int64 (reject floats, exponents, overflow) + if v, err := num.Int64(); err == nil { + return v, nil + } + debuglogger.Logf("invalid POST policy integer value: %s", string(raw)) + return 0, s3err.InvalidPolicyDocument.InvalidJSON() + } + + // AWS also accepts quoted integers here. + var s string + if err := json.Unmarshal(raw, &s); err == nil { + v, err := strconv.ParseInt(s, 10, 64) + if err != nil { + debuglogger.Logf("invalid POST policy quoted integer: %s", s) + return 0, s3err.InvalidPolicyDocument.ConditionFailed(string(rawCondition)) + } + return v, nil + } + + debuglogger.Logf("invalid POST policy integer JSON: %s", string(raw)) + return 0, s3err.InvalidPolicyDocument.InvalidJSON() +} + +// rawScalarToString converts a JSON scalar that is valid in policy conditions +// into its string representation. +func rawScalarToString(raw json.RawMessage) (string, error) { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return "", err + } + + switch x := v.(type) { + case string: + return x, nil + case json.Number: + return x.String(), nil + default: + return "", errors.New("unsupported type") + } +} diff --git a/auth/post_policy_test.go b/auth/post_policy_test.go new file mode 100644 index 00000000..4b68448b --- /dev/null +++ b/auth/post_policy_test.go @@ -0,0 +1,355 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "encoding/base64" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/s3err" +) + +func encodePolicyForTest(t *testing.T, expiration time.Time, conditions []any, rawBase64 bool) string { + t.Helper() + + policy := map[string]any{ + "expiration": expiration.UTC().Format(time.RFC3339), + "conditions": conditions, + } + + b, err := json.Marshal(policy) + assert.NoError(t, err) + + if rawBase64 { + return base64.RawStdEncoding.EncodeToString(b) + } + + return base64.StdEncoding.EncodeToString(b) +} + +func encodeRawPolicyJSONForTest(t *testing.T, rawJSON string) string { + t.Helper() + return base64.StdEncoding.EncodeToString([]byte(rawJSON)) +} + +func mustParsePolicyForTest(t *testing.T, encoded string) *POSTPolicy { + t.Helper() + p, err := ParsePOSTPolicyBase64(encoded) + assert.NoError(t, err) + return p +} + +func TestParsePOSTPolicyBase64_Success_AndEvaluate(t *testing.T) { + encoded := encodePolicyForTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "photos"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"}, + []any{"content-length-range", 1, 10}, + }, false) + + p := mustParsePolicyForTest(t, encoded) + + assert.NoError(t, p.Evaluate(PostPolicyEvalInput{ + Bucket: "photos", + Key: "uploads/image.jpg", + ContentLength: 5, + Fields: map[string]string{ + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "file": "ignored", + "policy": encoded, + "x-amz-signature": "ignored", + "x-ignore-meta": "ignored", + }, + })) +} + +func TestParsePOSTPolicyBase64_AcceptsRawBase64(t *testing.T) { + encoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + map[string]string{"bucket": "photos"}, + }, true) + + _, err := ParsePOSTPolicyBase64(encoded) + assert.Equal(t, error(nil), err) +} + +func TestParsePOSTPolicyBase64_ConcreteParseErrors(t *testing.T) { + floatRangeEncoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + []any{"content-length-range", 1.5, 10}, + }, false) + + unknownOpEncoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + []any{"contains", "$key", "uploads/"}, + }, false) + + badQuotedRangeEncoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + []any{"content-length-range", "abc", 10}, + }, false) + + tests := []struct { + name string + encoded string + expected error + }{ + { + name: "empty policy", + encoded: " ", + expected: s3err.InvalidPolicyDocument.EmptyPolicy(), + }, + { + name: "invalid base64", + encoded: "%%%not-base64%%%", + expected: s3err.InvalidPolicyDocument.InvalidBase64Encoding(), + }, + { + name: "invalid json", + encoded: encodeRawPolicyJSONForTest(t, `{"expiration":`), + expected: s3err.InvalidPolicyDocument.InvalidJSON(), + }, + { + name: "missing expiration", + encoded: encodeRawPolicyJSONForTest(t, `{ + "conditions":[{"bucket":"photos"}] + }`), + expected: s3err.InvalidPolicyDocument.MissingExpiration(), + }, + { + name: "missing conditions", + encoded: encodeRawPolicyJSONForTest(t, `{ + "expiration":"2100-01-01T00:00:00Z" + }`), + expected: s3err.InvalidPolicyDocument.MissingConditions(), + }, + { + name: "invalid expiration format", + encoded: encodeRawPolicyJSONForTest(t, `{ + "expiration":"not-a-time", + "conditions":[{"bucket":"photos"}] + }`), + expected: s3err.InvalidPolicyDocument.InvalidExpiration("not-a-time"), + }, + { + name: "expired policy", + encoded: encodePolicyForTest(t, time.Now().Add(-5*time.Minute), []any{ + map[string]string{"bucket": "photos"}, + }, false), + expected: s3err.InvalidPolicyDocument.PolicyExpired(), + }, + { + name: "unknown operation", + encoded: unknownOpEncoded, + expected: s3err.InvalidPolicyDocument.UnknownConditionOperation("contains"), + }, + { + name: "array condition missing op identifier", + encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + []any{}, + }, false), + expected: s3err.InvalidPolicyDocument.MissingConditionOperationIdentifier(), + }, + { + name: "eq wrong number of args", + encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + []any{"eq", "$key"}, + }, false), + expected: s3err.InvalidPolicyDocument.IncorrectConditionArgumentsNumber("eq"), + }, + { + name: "content-length-range wrong number of args", + encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + []any{"content-length-range", 1}, + }, false), + expected: s3err.InvalidPolicyDocument.IncorrectConditionArgumentsNumber("content-length-range"), + }, + { + name: "invalid simple condition value type", + encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + map[string]any{"bucket": 1}, + }, false), + expected: s3err.InvalidPolicyDocument.InvalidSimpleCondition(), + }, + { + name: "simple condition with multiple properties", + encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + map[string]string{"bucket": "photos", "acl": "private"}, + }, false), + expected: s3err.InvalidPolicyDocument.OnePropSimpleCondition(), + }, + { + name: "condition not object or list", + encoded: encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + 123, + }, false), + expected: s3err.InvalidPolicyDocument.InvalidCondition(), + }, + { + name: "content-length-range with float", + encoded: floatRangeEncoded, + expected: s3err.InvalidPolicyDocument.InvalidJSON(), + }, + { + name: "content-length-range with invalid quoted int", + encoded: badQuotedRangeEncoded, + expected: s3err.InvalidPolicyDocument.ConditionFailed(`["content-length-range","abc",10]`), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ParsePOSTPolicyBase64(tt.encoded) + assert.Equal(t, tt.expected, err) + }) + } +} + +func TestPOSTPolicyEvaluate_ConcretePolicyRejections(t *testing.T) { + encoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + map[string]string{"bucket": "photos"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"}, + []any{"content-length-range", 2, 4}, + }, false) + + p := mustParsePolicyForTest(t, encoded) + + tests := []struct { + name string + input PostPolicyEvalInput + expected error + }{ + { + name: "extra field", + input: PostPolicyEvalInput{ + Bucket: "photos", + Key: "uploads/image.jpg", + ContentLength: 3, + Fields: map[string]string{ + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "acl": "private", + }, + }, + expected: s3err.InvalidPolicyDocument.ExtraInputField("acl"), + }, + { + name: "bucket mismatch", + input: PostPolicyEvalInput{ + Bucket: "other-bucket", + Key: "uploads/image.jpg", + ContentLength: 3, + Fields: map[string]string{ + "x-amz-algorithm": "AWS4-HMAC-SHA256", + }, + }, + expected: s3err.InvalidPolicyDocument.ConditionFailed(`["eq", "$bucket", "photos"]`), + }, + { + name: "key prefix mismatch", + input: PostPolicyEvalInput{ + Bucket: "photos", + Key: "tmp/image.jpg", + ContentLength: 3, + Fields: map[string]string{ + "x-amz-algorithm": "AWS4-HMAC-SHA256", + }, + }, + expected: s3err.InvalidPolicyDocument.ConditionFailed(`["starts-with","$key","uploads/"]`), + }, + { + name: "missing required field", + input: PostPolicyEvalInput{ + Bucket: "photos", + Key: "uploads/image.jpg", + ContentLength: 3, + Fields: map[string]string{}, + }, + expected: s3err.InvalidPolicyDocument.ConditionFailed(`["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"]`), + }, + { + name: "content too large", + input: PostPolicyEvalInput{ + Bucket: "photos", + Key: "uploads/image.jpg", + ContentLength: 10, + Fields: map[string]string{ + "x-amz-algorithm": "AWS4-HMAC-SHA256", + }, + }, + expected: s3err.GetAPIError(s3err.ErrEntityTooLarge), + }, + { + name: "content too small", + input: PostPolicyEvalInput{ + Bucket: "photos", + Key: "uploads/image.jpg", + ContentLength: 1, + Fields: map[string]string{ + "x-amz-algorithm": "AWS4-HMAC-SHA256", + }, + }, + expected: s3err.GetAPIError(s3err.ErrEntityTooSmall), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := p.Evaluate(tt.input) + assert.Equal(t, tt.expected, err) + }) + } +} + +func TestPOSTPolicyEvaluate_IgnoresAllowedCoverageFields(t *testing.T) { + encoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + map[string]string{"bucket": "photos"}, + []any{"starts-with", "$key", "uploads/"}, + }, false) + + p := mustParsePolicyForTest(t, encoded) + + err := p.Evaluate(PostPolicyEvalInput{ + Bucket: "photos", + Key: "uploads/image.jpg", + ContentLength: 3, + Fields: map[string]string{ + "file": "ignored", + "policy": "ignored", + "x-amz-signature": "ignored", + "x-ignore-meta": "ignored", + }, + }) + assert.Equal(t, error(nil), err) +} + +func TestPOSTPolicyEvaluate_ContentTypeStartsWithList_AllEntriesMustMatch(t *testing.T) { + encoded := encodePolicyForTest(t, time.Now().Add(5*time.Minute), []any{ + map[string]string{"bucket": "photos"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"starts-with", "$Content-Type", "image/"}, + }, false) + + p := mustParsePolicyForTest(t, encoded) + + err := p.Evaluate(PostPolicyEvalInput{ + Bucket: "photos", + Key: "uploads/image.jpg", + ContentLength: 3, + Fields: map[string]string{ + "content-type": "image/png,text/plain", + }, + }) + assert.Equal(t, s3err.InvalidPolicyDocument.ConditionFailed(`["starts-with","$Content-Type","image/"]`), err) +} diff --git a/metrics/actions.go b/metrics/actions.go index 823c3818..b8c13149 100644 --- a/metrics/actions.go +++ b/metrics/actions.go @@ -117,6 +117,7 @@ var ( ActionDeleteBucketWebsite = "s3_DeleteBucketWebsite" ActionGetBucketPolicyStatus = "s3_GetBucketPolicyStatus" ActionGetBucketLocation = "s3_GetBucketLocation" + ActionPostObject = "s3_PostObject" // Admin actions ActionAdminCreateUser = "admin_CreateUser" @@ -504,4 +505,8 @@ func init() { Name: "GetBucketLocation", Service: "s3", } + ActionMap[ActionPostObject] = Action{ + Name: "PostObject", + Service: "s3", + } } diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go index 52819da9..24e624f9 100644 --- a/s3api/controllers/bucket-post.go +++ b/s3api/controllers/bucket-post.go @@ -16,6 +16,8 @@ package controllers import ( "encoding/xml" + "net/http" + "net/url" "strings" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -23,6 +25,7 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3api/middlewares" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3event" @@ -93,3 +96,206 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { }, }, err } + +func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) { + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + parsed := utils.ContextKeyObjectPostResult.Get(ctx).(middlewares.PostObjectResult) + bucket := ctx.Params("bucket") + contentType := parsed.Fields["content-type"] + contentEncoding := parsed.Fields["content-encoding"] + contentDisposition := parsed.Fields["content-disposition"] + contentLanguage := parsed.Fields["content-language"] + cacheControl := parsed.Fields["cache-control"] + expires := parsed.Fields["expires"] + + key, ok := parsed.Fields["key"] + if !ok || key == "" { + debuglogger.Logf("missing object key") + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.PostAuth.MissingField("key") + } + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutObjectAction, + IsPublicRequest: IsBucketPublic, + DisableACL: c.disableACL, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // parse POST policy — absent for anonymous uploads to public buckets + policyBase64 := parsed.Fields["policy"] + if policyBase64 != "" { + policy, err := auth.ParsePOSTPolicyBase64(policyBase64) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // Evaluate post policy + err = policy.Evaluate(auth.PostPolicyEvalInput{ + Bucket: bucket, + Key: key, + ContentLength: parsed.ContentLength, + Fields: parsed.Fields, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + } + + // convert object tagging from raw XML to Query string + // to pass PutObject, which expects the tagging to be a query string + var tagging string + if taggingXML, ok := parsed.Fields["tagging"]; ok { + tagging, err = utils.ConvertTaggingXMLToQueryString([]byte(taggingXML)) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + } + + // parse checksum headers + checksums, err := utils.ParseCalculatedChecksumFields(parsed.Fields) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // extract metadata + metadata, err := utils.ExtractMetadataFromFields(parsed.Fields) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.PutObject(ctx.Context(), s3response.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ContentType: &contentType, + ContentEncoding: &contentEncoding, + ContentDisposition: &contentDisposition, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, + Body: parsed.FileRdr, + ContentLength: &parsed.ContentLength, + Tagging: &tagging, + Metadata: metadata, + ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + if successActionRedirect, ok := parsed.Fields["success_action_redirect"]; ok { + u, err := url.Parse(successActionRedirect) + if err == nil { + q := u.Query() + q.Set("bucket", bucket) + q.Set("key", key) + q.Set("etag", res.ETag) + u.RawQuery = q.Encode() + redirectURI := u.String() + + return &Response{ + Headers: map[string]*string{ + "Location": &redirectURI, + }, + MetaOpts: &MetaOptions{ + ContentLength: parsed.FileRdr.Length(), + BucketOwner: parsedAcl.Owner, + ObjectETag: &res.ETag, + ObjectSize: parsed.FileRdr.Length(), + EventName: s3event.EventObjectCreatedPost, + Status: http.StatusSeeOther, + }, + }, nil + } + } + + respStatus := http.StatusNoContent + var respBody any + location := utils.GenerateObjectLocation(ctx, c.virtualDomain, bucket, key) + + if successStatus, ok := parsed.Fields["success_action_status"]; ok { + switch successStatus { + case "200": + respStatus = http.StatusOK + case "201": + respStatus = http.StatusCreated + respBody = &s3response.PostResponse{ + Bucket: bucket, + Key: key, + ETag: res.ETag, + Location: location, + } + } + } + + return &Response{ + Headers: map[string]*string{ + "Etag": &res.ETag, + "Location": &location, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-version-id": utils.GetStringPtr(res.VersionID), + }, + Data: respBody, + MetaOpts: &MetaOptions{ + ContentLength: parsed.FileRdr.Length(), + BucketOwner: parsedAcl.Owner, + ObjectETag: &res.ETag, + ObjectSize: parsed.FileRdr.Length(), + EventName: s3event.EventObjectCreatedPost, + Status: respStatus, + }, + }, nil +} diff --git a/s3api/controllers/bucket-post_test.go b/s3api/controllers/bucket-post_test.go index 40cf2879..a5e5013e 100644 --- a/s3api/controllers/bucket-post_test.go +++ b/s3api/controllers/bucket-post_test.go @@ -16,12 +16,20 @@ package controllers import ( "context" + "encoding/base64" + "encoding/json" "encoding/xml" + "io" + "net/http" + "strings" "testing" + "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/middlewares" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3event" @@ -163,3 +171,528 @@ func TestS3ApiController_DeleteObjects(t *testing.T) { }) } } + +// mockMpFileReader wraps an io.Reader and satisfies utils.MpFileReader. +// It tracks the number of bytes delivered to callers so that Length returns +// the same value that finalFileReader.Length would return after a real upload. +type mockMpFileReader struct { + r io.Reader + bytesRead int64 +} + +func (m *mockMpFileReader) Read(p []byte) (int, error) { + n, err := m.r.Read(p) + m.bytesRead += int64(n) + return n, err +} + +func (m *mockMpFileReader) Length() int64 { return m.bytesRead } + +func newMockFileReader(content string) *mockMpFileReader { + return &mockMpFileReader{r: strings.NewReader(content)} +} + +func TestS3ApiController_POSTObject(t *testing.T) { + encodePOSTPolicyForControllerTest := func(t *testing.T, expiration time.Time, conditions []any) string { + t.Helper() + + policy := map[string]any{ + "expiration": expiration.UTC().Format(time.RFC3339), + "conditions": conditions, + } + + b, err := json.Marshal(policy) + assert.NoError(t, err) + + return base64.StdEncoding.EncodeToString(b) + } + postObjectLocalsForTest := func(parsed middlewares.PostObjectResult) map[utils.ContextKey]any { + return map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + utils.ContextKeyRegion: "us-east-1", + utils.ContextKeyObjectPostResult: parsed, + } + } + marshalObjectTaggingForControllerTest := func(t *testing.T, tags []s3response.Tag) string { + t.Helper() + + data, err := xml.Marshal(s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: tags, + }, + }) + assert.NoError(t, err) + + return string(data) + } + + validTaggingXML := marshalObjectTaggingForControllerTest(t, []s3response.Tag{ + {Key: "project", Value: "alpha team"}, + }) + baseFields := map[string]string{ + "key": "uploads/photo.jpg", + "file": "ignored", + "x-amz-signature": "ignored", + } + basePolicy := encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + }) + baseFields["policy"] = basePolicy + + location := "http://example.com/bucket/uploads%2Fphoto.jpg" + + anonFields := map[string]string{ + "key": "uploads/anon.bin", + "file": "ignored", + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "missing key", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "policy": basePolicy, + "file": "ignored", + "x-amz-signature": "ignored", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.PostAuth.MissingField("key"), + }, + }, + { + name: "verify access fails", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: false, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "user", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "user", + Role: auth.RoleUser, + }, + utils.ContextKeyRegion: "us-east-1", + utils.ContextKeyObjectPostResult: middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "key", + }, + }, + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "user", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid policy", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": "%%%not-base64%%%", + "file": "ignored", + "x-amz-signature": "ignored", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.InvalidPolicyDocument.InvalidBase64Encoding(), + }, + }, + { + name: "policy evaluation fails on extra field", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": basePolicy, + "file": "ignored", + "x-amz-signature": "ignored", + "unexpected": "value", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.InvalidPolicyDocument.ExtraInputField("unexpected"), + }, + }, + { + name: "invalid tagging xml", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$tagging", "invalid-xml"}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "tagging": "invalid-xml", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "invalid checksum fields", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$x-amz-checksum-crc32", "invalid_base64_string"}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "x-amz-checksum-crc32": "invalid_base64_string", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"), + }, + }, + { + name: "metadata too large", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"starts-with", "$x-amz-meta-big", ""}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "x-amz-meta-big": strings.Repeat("a", 2050), + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMetadataTooLarge), + }, + }, + { + name: "backend returns error", + input: testInput{ + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: baseFields, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful redirect response", + input: testInput{ + beRes: s3response.PutObjectOutput{ + ETag: "etag-123", + VersionID: "vid-123", + }, + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$success_action_redirect", "https://client.example/upload-complete"}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "success_action_redirect": "https://client.example/upload-complete", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "Location": utils.GetStringPtr("https://client.example/upload-complete?bucket=bucket&etag=etag-123&key=uploads%2Fphoto.jpg"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ContentLength: int64(len("payload")), + ObjectETag: utils.GetStringPtr("etag-123"), + ObjectSize: int64(len("payload")), + EventName: s3event.EventObjectCreatedPost, + Status: 303, + }, + }, + }, + }, + { + name: "successful created response", + input: testInput{ + beRes: s3response.PutObjectOutput{ + ETag: "etag-123", + VersionID: "vid-123", + ChecksumCRC32: utils.GetStringPtr("crc32-out"), + ChecksumCRC32C: utils.GetStringPtr("crc32c-out"), + ChecksumSHA1: utils.GetStringPtr("sha1-out"), + ChecksumSHA256: utils.GetStringPtr("sha256-out"), + ChecksumCRC64NVME: utils.GetStringPtr("crc64-out"), + ChecksumType: types.ChecksumTypeComposite, + }, + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$success_action_status", "201"}, + []any{"eq", "$tagging", validTaggingXML}, + []any{"eq", "$x-amz-meta-owner", "alice"}, + []any{"eq", "$x-amz-checksum-crc32", "ww2FVQ=="}, + []any{"eq", "$cache-control", "max-age=60"}, + []any{"eq", "$content-type", "image/jpeg"}, + []any{"eq", "$content-disposition", "inline"}, + []any{"eq", "$content-encoding", "gzip"}, + []any{"eq", "$content-language", "en-US"}, + []any{"eq", "$expires", "Fri, 21 Mar 2026 00:00:00 GMT"}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "success_action_status": "201", + "tagging": validTaggingXML, + "x-amz-meta-owner": "alice", + "x-amz-checksum-crc32": "ww2FVQ==", + "cache-control": "max-age=60", + "content-type": "image/jpeg", + "content-disposition": "inline", + "content-encoding": "gzip", + "content-language": "en-US", + "expires": "Fri, 21 Mar 2026 00:00:00 GMT", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "Etag": utils.GetStringPtr("etag-123"), + "Location": &location, + "x-amz-checksum-crc32": utils.GetStringPtr("crc32-out"), + "x-amz-checksum-crc32c": utils.GetStringPtr("crc32c-out"), + "x-amz-checksum-crc64nvme": utils.GetStringPtr("crc64-out"), + "x-amz-checksum-sha1": utils.GetStringPtr("sha1-out"), + "x-amz-checksum-sha256": utils.GetStringPtr("sha256-out"), + "x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)), + "x-amz-version-id": utils.GetStringPtr("vid-123"), + }, + Data: &s3response.PostResponse{ + Bucket: "bucket", + Key: "uploads/photo.jpg", + ETag: "etag-123", + Location: location, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ContentLength: int64(len("payload")), + ObjectETag: utils.GetStringPtr("etag-123"), + ObjectSize: int64(len("payload")), + EventName: s3event.EventObjectCreatedPost, + Status: 201, + }, + }, + }, + }, + { + name: "anonymous upload succeeds without policy", + input: testInput{ + beRes: s3response.PutObjectOutput{ + ETag: "etag-anon", + }, + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: anonFields, + FileRdr: newMockFileReader("anon-payload"), + ContentLength: int64(len("anon-payload")), + }), + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "Etag": utils.GetStringPtr("etag-anon"), + "Location": utils.GetStringPtr("http://example.com/bucket/uploads%2Fanon.bin"), + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-checksum-type": nil, + "x-amz-version-id": utils.GetStringPtr(""), // empty, not nil — controller always returns &res.VersionID + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ContentLength: int64(len("anon-payload")), + ObjectETag: utils.GetStringPtr("etag-anon"), + ObjectSize: int64(len("anon-payload")), + EventName: s3event.EventObjectCreatedPost, + Status: http.StatusNoContent, + }, + }, + }, + }, + { + name: "anonymous upload with policy is evaluated", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/anon.bin", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + // key condition intentionally omitted -> ExtraInputField for "key" + }), + "file": "ignored", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.InvalidPolicyDocument.ExtraInputField("key"), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) { + if tt.input.beErr != nil { + return s3response.PutObjectOutput{}, tt.input.beErr + } + + // Drain the body as a real backend would, so that FileRdr.Length() + // reflects the actual bytes written after PutObject returns. + body, err := io.ReadAll(putObjectInput.Body) + assert.NoError(t, err) + + if tt.name == "anonymous upload succeeds without policy" { + assert.Equal(t, "uploads/anon.bin", *putObjectInput.Key) + assert.Equal(t, "anon-payload", string(body)) + } + + if tt.name == "successful created response" { + assert.Equal(t, "bucket", *putObjectInput.Bucket) + assert.Equal(t, "uploads/photo.jpg", *putObjectInput.Key) + assert.Equal(t, "image/jpeg", *putObjectInput.ContentType) + assert.Equal(t, "gzip", *putObjectInput.ContentEncoding) + assert.Equal(t, "inline", *putObjectInput.ContentDisposition) + assert.Equal(t, "en-US", *putObjectInput.ContentLanguage) + assert.Equal(t, "max-age=60", *putObjectInput.CacheControl) + assert.Equal(t, "Fri, 21 Mar 2026 00:00:00 GMT", *putObjectInput.Expires) + assert.Equal(t, int64(len("payload")), *putObjectInput.ContentLength) + assert.Equal(t, "project=alpha+team", *putObjectInput.Tagging) + assert.Equal(t, map[string]string{"owner": "alice"}, putObjectInput.Metadata) + assert.Equal(t, utils.GetStringPtr("ww2FVQ=="), putObjectInput.ChecksumCRC32) + assert.Equal(t, "payload", string(body)) + } + + return tt.input.beRes.(s3response.PutObjectOutput), nil + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.POSTObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }, + ) + }) + } +} diff --git a/s3api/controllers/bucket-put_test.go b/s3api/controllers/bucket-put_test.go index aad784c2..58e258c1 100644 --- a/s3api/controllers/bucket-put_test.go +++ b/s3api/controllers/bucket-put_test.go @@ -32,7 +32,7 @@ import ( ) func TestS3ApiController_PutBucketTagging(t *testing.T) { - validTaggingBody, err := xml.Marshal(s3response.TaggingInput{ + validTaggingBody, err := xml.Marshal(s3response.Tagging{ TagSet: s3response.TagSet{ Tags: []s3response.Tag{ { diff --git a/s3api/controllers/object-put_test.go b/s3api/controllers/object-put_test.go index 01aa1164..f82e544b 100644 --- a/s3api/controllers/object-put_test.go +++ b/s3api/controllers/object-put_test.go @@ -34,7 +34,7 @@ import ( func TestS3ApiController_PutObjectTagging(t *testing.T) { validTaggingBody, err := xml.Marshal( - s3response.TaggingInput{ + s3response.Tagging{ TagSet: s3response.TagSet{ Tags: []s3response.Tag{ { diff --git a/s3api/middlewares/object-post-auth.go b/s3api/middlewares/object-post-auth.go new file mode 100644 index 00000000..1a83809e --- /dev/null +++ b/s3api/middlewares/object-post-auth.go @@ -0,0 +1,187 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package middlewares + +import ( + "bytes" + "mime" + "strconv" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +const ( + formFieldPolicy = "policy" + formFieldAlgorithm = "x-amz-algorithm" + formFieldCredential = "x-amz-credential" + formFieldDate = "x-amz-date" + formFieldSignature = "x-amz-signature" + + aws4HMACSHA256 = "AWS4-HMAC-SHA256" +) + +type PostObjectResult struct { + ContentLength int64 + // FileRdr streams the file payload. Length() reports the exact number of + // file-content bytes read after the backend has consumed the body. + FileRdr utils.MpFileReader + Fields map[string]string +} + +func AuthorizePostObject(root RootUserConfig, iam auth.IAMService, region string) fiber.Handler { + acct := accounts{root: root, iam: iam} + + return func(ctx *fiber.Ctx) error { + contentLengthStr := ctx.Get("Content-Length") + reqContentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + debuglogger.Logf("invalid POST object Content-Length %q: %v", contentLengthStr, err) + return s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + mediaType, params, err := mime.ParseMediaType(ctx.Get("Content-Type")) + if err != nil || mediaType != fiber.MIMEMultipartForm { + debuglogger.Logf("invalid POST object Content-Type %q: mediaType=%q err=%v", ctx.Get("Content-Type"), mediaType, err) + return s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + + boundary := params["boundary"] + if boundary == "" { + debuglogger.Logf("missing multipart boundary in POST object request") + return s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + bodyRdr := ctx.Request().BodyStream() + if bodyRdr == nil { + bodyRdr = bytes.NewReader(ctx.Body()) + } + + mpParser, err := utils.NewMultipartParser(bodyRdr, boundary, reqContentLength) + if err != nil { + return err + } + + result, err := mpParser.Parse() + if err != nil { + return err + } + + fields := result.Fields + + policyB64 := fields[formFieldPolicy] + algorithm := fields[formFieldAlgorithm] + credentialStr := fields[formFieldCredential] + amzDate := fields[formFieldDate] + signatureHex := fields[formFieldSignature] + + // Determine if the request carries form-based credentials. + // A request is considered signed if ANY of the five auth fields is + // present; in that case ALL of them are required. + hasAnyAuthField := policyB64 != "" || algorithm != "" || credentialStr != "" || amzDate != "" || signatureHex != "" + + if hasAnyAuthField { + // Signed POST Object — validate every required auth field. + for _, field := range []struct { + key string + value string + }{ + {formFieldPolicy, policyB64}, + {formFieldAlgorithm, algorithm}, + {formFieldCredential, credentialStr}, + {formFieldDate, amzDate}, + {formFieldSignature, signatureHex}, + } { + if field.value == "" { + debuglogger.Logf("missing required POST object field: %s", field.key) + return s3err.PostAuth.MissingField(field.key) + } + } + + if algorithm != aws4HMACSHA256 { + debuglogger.Logf("unsupported POST object signing algorithm: %s", algorithm) + return s3err.GetAPIError(s3err.ErrOnlyAws4HmacSha256) + } + + // Parse the date and check the date validity + tdate, err := time.Parse(iso8601Format, amzDate) + if err != nil { + debuglogger.Logf("invalid POST object x-amz-date %q: %v", amzDate, err) + return s3err.GetAPIError(s3err.ErrInvalidDateHeader) + } + + // Validate the dates difference + // TODO: Seems s3 doesn't validate this + err = utils.ValidateDate(tdate) + if err != nil { + return err + } + + creds, err := utils.ParseCredentials(credentialStr, s3err.PostAuth) + if err != nil { + return err + } + + if region != creds.Region { + debuglogger.Logf("incorrect POST object credential region: got %q want %q", creds.Region, region) + return s3err.MalformedAuth.IncorrectRegion(region, creds.Region) + } + + account, err := acct.getAccount(creds.Access) + if err == auth.ErrNoSuchUser { + debuglogger.Logf("POST object access key not found: %s", creds.Access) + return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID) + } + if err != nil { + debuglogger.Logf("failed to resolve POST object account %q: %v", creds.Access, err) + return err + } + + utils.ContextKeyAccount.Set(ctx, account) + utils.ContextKeyIsRoot.Set(ctx, account.Access == root.Access) + + expectedSig, err := utils.SignPostPolicy(policyB64, creds.Date, region, account.Secret) + if err != nil { + return err + } + + if expectedSig != signatureHex { + debuglogger.Logf("POST object signature mismatch: expected %s got %s", expectedSig, signatureHex) + return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch) + } + + // Mark this request as authenticated so that + // AuthorizePublicBucketAccess (running after this middleware) + // skips its anonymous-access check. + utils.ContextKeyAuthenticated.Set(ctx, true) + } + // else: anonymous POST Object — no credentials in form fields. + // AuthorizePublicBucketAccess will verify public bucket access next. + + utils.ContextKeyObjectPostResult.Set(ctx, + PostObjectResult{ + Fields: fields, + FileRdr: result.FileRdr, + ContentLength: result.ContentLength, + }, + ) + + return nil + } +} diff --git a/s3api/middlewares/object-post-auth_test.go b/s3api/middlewares/object-post-auth_test.go new file mode 100644 index 00000000..180d12ab --- /dev/null +++ b/s3api/middlewares/object-post-auth_test.go @@ -0,0 +1,292 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package middlewares + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strings" + "testing" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +// chainHandlers mimics controllers.ProcessHandlers: it calls each handler in +// sequence and stops on the first error. AuthorizePostObject (like all +// versitygw middlewares) returns nil without calling c.Next(), so they must +// be chained explicitly rather than relying on fiber's c.Next() mechanism. +func chainHandlers(handlers ...fiber.Handler) fiber.Handler { + return func(c *fiber.Ctx) error { + for _, h := range handlers { + if err := h(c); err != nil { + return err + } + } + return nil + } +} + +// postObjectTestApp creates a fiber app that chains AuthorizePostObject with +// the provided follow-up handler on POST /:bucket. +func postObjectTestApp(root RootUserConfig, region string, next fiber.Handler) *fiber.App { + app := fiber.New(fiber.Config{ + ErrorHandler: func(c *fiber.Ctx, err error) error { + if apiErr, ok := err.(s3err.APIError); ok { + return c.Status(apiErr.HTTPStatusCode).SendString(apiErr.Code) + } + return c.Status(500).SendString(err.Error()) + }, + }) + app.Post("/:bucket", chainHandlers(AuthorizePostObject(root, nil, region), next)) + return app +} + +// buildMultipartBody returns a multipart/form-data body and its boundary. +// The given fields are written as form fields; fileContent is written as the +// "file" part. +func buildMultipartBody(t *testing.T, fields map[string]string, fileContent string) ([]byte, string) { + t.Helper() + + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + for key, value := range fields { + assert.NoError(t, w.WriteField(key, value)) + } + + fw, err := w.CreateFormFile("file", "upload.bin") + assert.NoError(t, err) + _, err = io.WriteString(fw, fileContent) + assert.NoError(t, err) + + assert.NoError(t, w.Close()) + return buf.Bytes(), w.Boundary() +} + +// encodePOSTPolicy encodes a minimal valid policy expiring 15 minutes in the +// future with the supplied conditions. +func encodePOSTPolicy(t *testing.T, conditions []any) string { + t.Helper() + + policy := map[string]any{ + "expiration": time.Now().Add(15 * time.Minute).UTC().Format(time.RFC3339), + "conditions": conditions, + } + b, err := json.Marshal(policy) + assert.NoError(t, err) + return base64.StdEncoding.EncodeToString(b) +} + +func makePostRequest(t *testing.T, body []byte, boundary string) *http.Request { + t.Helper() + + req, err := http.NewRequest(http.MethodPost, "/mybucket", bytes.NewReader(body)) + assert.NoError(t, err) + req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%s", boundary)) + // Set both the header and the int64 field: app.Test() serialises the + // request via req.Write() which uses req.ContentLength; fasthttp then + // re-reads it as the Content-Length header. + req.ContentLength = int64(len(body)) + return req +} + +// TestAuthorizePostObject_AnonymousRequest verifies that a POST with no auth +// fields succeeds: PostObjectResult is populated and ContextKeyAuthenticated +// is NOT set. +func TestAuthorizePostObject_AnonymousRequest(t *testing.T) { + var ( + gotResult PostObjectResult + gotAuthenticated bool + ) + + app := postObjectTestApp( + RootUserConfig{Access: "root", Secret: "rootsecret"}, + "us-east-1", + func(c *fiber.Ctx) error { + gotResult = utils.ContextKeyObjectPostResult.Get(c).(PostObjectResult) + gotAuthenticated = utils.ContextKeyAuthenticated.IsSet(c) + return c.SendStatus(http.StatusOK) + }, + ) + + body, boundary := buildMultipartBody(t, map[string]string{ + "key": "uploads/photo.jpg", + }, "file-content") + + resp, err := app.Test(makePostRequest(t, body, boundary)) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "uploads/photo.jpg", gotResult.Fields["key"]) + assert.False(t, gotAuthenticated, "anonymous request must not set ContextKeyAuthenticated") +} + +// TestAuthorizePostObject_AnonymousRequest_SetsPostObjectResult verifies that +// ContentType and FileRdr are populated for an anonymous upload. +func TestAuthorizePostObject_AnonymousRequest_SetsPostObjectResult(t *testing.T) { + var gotResult PostObjectResult + + app := postObjectTestApp( + RootUserConfig{Access: "root", Secret: "rootsecret"}, + "us-east-1", + func(c *fiber.Ctx) error { + gotResult = utils.ContextKeyObjectPostResult.Get(c).(PostObjectResult) + return c.SendStatus(http.StatusOK) + }, + ) + + body, boundary := buildMultipartBody(t, map[string]string{ + "key": "uploads/hello.txt", + "Content-Type": "text/plain", + }, "hello world") + + resp, err := app.Test(makePostRequest(t, body, boundary)) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "uploads/hello.txt", gotResult.Fields["key"]) + assert.NotNil(t, gotResult.FileRdr) +} + +// TestAuthorizePostObject_SignedRequest verifies that a correctly signed POST +// succeeds and sets ContextKeyAuthenticated. +func TestAuthorizePostObject_SignedRequest(t *testing.T) { + const ( + region = "us-east-1" + accessKey = "testaccess" + secretKey = "testsecret" + ) + + now := time.Now().UTC() + dateShort := now.Format("20060102") + dateLong := now.Format("20060102T150405Z") + + credential := fmt.Sprintf("%s/%s/%s/s3/aws4_request", accessKey, dateShort, region) + policyB64 := encodePOSTPolicy(t, []any{ + map[string]string{"bucket": "mybucket"}, + []any{"starts-with", "$key", "uploads/"}, + }) + sig, err := utils.SignPostPolicy(policyB64, dateShort, region, secretKey) + assert.NoError(t, err) + + var gotAuthenticated bool + + app := postObjectTestApp( + RootUserConfig{Access: accessKey, Secret: secretKey}, + region, + func(c *fiber.Ctx) error { + gotAuthenticated = utils.ContextKeyAuthenticated.IsSet(c) + return c.SendStatus(http.StatusOK) + }, + ) + + body, boundary := buildMultipartBody(t, map[string]string{ + "key": "uploads/photo.jpg", + "policy": policyB64, + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "x-amz-credential": credential, + "x-amz-date": dateLong, + "x-amz-signature": sig, + }, "file-content") + + resp, err := app.Test(makePostRequest(t, body, boundary)) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.True(t, gotAuthenticated, "signed request must set ContextKeyAuthenticated") +} + +// TestAuthorizePostObject_SignedRequest_WrongSignature verifies that a signed +// POST with a bad signature is rejected. +func TestAuthorizePostObject_SignedRequest_WrongSignature(t *testing.T) { + const ( + region = "us-east-1" + accessKey = "testaccess" + secretKey = "testsecret" + ) + + now := time.Now().UTC() + dateShort := now.Format("20060102") + dateLong := now.Format("20060102T150405Z") + + credential := fmt.Sprintf("%s/%s/%s/s3/aws4_request", accessKey, dateShort, region) + policyB64 := encodePOSTPolicy(t, []any{ + map[string]string{"bucket": "mybucket"}, + }) + + app := postObjectTestApp( + RootUserConfig{Access: accessKey, Secret: secretKey}, + region, + func(c *fiber.Ctx) error { return c.SendStatus(http.StatusOK) }, + ) + + body, boundary := buildMultipartBody(t, map[string]string{ + "key": "uploads/photo.jpg", + "policy": policyB64, + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "x-amz-credential": credential, + "x-amz-date": dateLong, + "x-amz-signature": "baadsignature00000000000", + }, "file-content") + + resp, err := app.Test(makePostRequest(t, body, boundary)) + assert.NoError(t, err) + assert.Equal(t, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch).HTTPStatusCode, resp.StatusCode) +} + +// TestAuthorizePostObject_PartialAuthFields_ReturnsError verifies that +// providing only some auth fields (e.g. x-amz-algorithm only) is rejected. +func TestAuthorizePostObject_PartialAuthFields_ReturnsError(t *testing.T) { + app := postObjectTestApp( + RootUserConfig{Access: "root", Secret: "rootsecret"}, + "us-east-1", + func(c *fiber.Ctx) error { return c.SendStatus(http.StatusOK) }, + ) + + // Only algorithm is provided — credential, date, policy, signature absent. + body, boundary := buildMultipartBody(t, map[string]string{ + "key": "uploads/photo.jpg", + "x-amz-algorithm": "AWS4-HMAC-SHA256", + }, "file-content") + + resp, err := app.Test(makePostRequest(t, body, boundary)) + assert.NoError(t, err) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) +} + +// TestAuthorizePostObject_InvalidContentType_ReturnsError verifies that a +// non-multipart Content-Type is rejected. +func TestAuthorizePostObject_InvalidContentType_ReturnsError(t *testing.T) { + app := postObjectTestApp( + RootUserConfig{Access: "root", Secret: "rootsecret"}, + "us-east-1", + func(c *fiber.Ctx) error { return c.SendStatus(http.StatusOK) }, + ) + + req, err := http.NewRequest(http.MethodPost, "/mybucket", strings.NewReader("body")) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Length", "4") + + resp, err := app.Test(req) + assert.NoError(t, err) + assert.Equal(t, s3err.GetAPIError(s3err.ErrPreconditionFailed).HTTPStatusCode, resp.StatusCode) +} diff --git a/s3api/middlewares/presign-auth.go b/s3api/middlewares/presign-auth.go index 6bebafdb..8c1754b4 100644 --- a/s3api/middlewares/presign-auth.go +++ b/s3api/middlewares/presign-auth.go @@ -45,7 +45,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region // otherwise the middleware will return the caucht error utils.ContextKeyAuthenticated.Set(ctx, true) - authData, err := utils.ParsePresignedURIParts(ctx) + authData, err := utils.ParsePresignedURIParts(ctx, region) if err != nil { return err } diff --git a/s3api/middlewares/public-bucket.go b/s3api/middlewares/public-bucket.go index ba7dce52..f0fd37fa 100644 --- a/s3api/middlewares/public-bucket.go +++ b/s3api/middlewares/public-bucket.go @@ -33,7 +33,7 @@ import ( func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPermission auth.Action, permission auth.Permission, region string, streamBody bool) fiber.Handler { return func(ctx *fiber.Ctx) error { // skip for authenticated requests - if utils.IsPresignedURLAuth(ctx) || ctx.Get("Authorization") != "" { + if utils.IsPresignedURLAuth(ctx) || ctx.Get("Authorization") != "" || utils.ContextKeyAuthenticated.IsSet(ctx) { return nil } diff --git a/s3api/router.go b/s3api/router.go index 54ceb274..3bd59252 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -1092,6 +1092,18 @@ func (sa *S3ApiRouter) Init() { middlewares.ParseAcl(sa.be), )) + bucketRouter.Post("", + controllers.ProcessHandlers( + ctrl.POSTObject, + metrics.ActionPostObject, + services, + middlewares.BucketObjectNameValidator(), + middlewares.AuthorizePostObject(sa.root, sa.iam, sa.region), + middlewares.AuthorizePublicBucketAccess(sa.be, metrics.ActionPostObject, auth.PutObjectAction, auth.PermissionWrite, sa.region, false), + middlewares.ApplyBucketCORS(sa.be, sa.corsAllowOrigin), + middlewares.ParseAcl(sa.be), + )) + // object HEAD operation is not allowed with copy source objectRouter.Head("/", middlewares.MatchHeader("X-Amz-Copy-Source"), diff --git a/s3api/utils/auth-reader.go b/s3api/utils/auth-reader.go index 6c7e5b4d..9cf24a8a 100644 --- a/s3api/utils/auth-reader.go +++ b/s3api/utils/auth-reader.go @@ -15,6 +15,9 @@ package utils import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io" @@ -226,23 +229,13 @@ func ParseAuthorization(authorization string) (AuthData, error) { switch key { case "Credential": - creds := strings.Split(value, "/") - if len(creds) != 5 { - return a, s3err.MalformedAuth.MalformedCredential() - } - if creds[3] != "s3" { - return a, s3err.MalformedAuth.IncorrectService(creds[3]) - } - if creds[4] != "aws4_request" { - return a, s3err.MalformedAuth.InvalidTerminal(creds[4]) - } - _, err := time.Parse(yyyymmdd, creds[1]) + creds, err := ParseCredentials(value, s3err.MalformedAuth) if err != nil { - return a, s3err.MalformedAuth.InvalidDateFormat(creds[1]) + return a, err } - access = creds[0] - date = creds[1] - region = creds[2] + access = creds.Access + date = creds.Date + region = creds.Region case "SignedHeaders": signedHeaders = value case "Signature": @@ -260,6 +253,41 @@ func ParseAuthorization(authorization string) (AuthData, error) { }, nil } +type CredentialsScope struct { + Access string + Date string + Region string +} + +type CredsError interface { + MalformedCredential() s3err.APIError + IncorrectService(string) s3err.APIError + IncorrectTerminal(string) s3err.APIError + InvalidDateFormat(string) s3err.APIError +} + +func ParseCredentials(input string, errHandler CredsError) (*CredentialsScope, error) { + creds := strings.Split(input, "/") + if len(creds) != 5 { + return nil, errHandler.MalformedCredential() + } + if creds[3] != "s3" { + return nil, errHandler.IncorrectService(creds[3]) + } + if creds[4] != "aws4_request" { + return nil, errHandler.IncorrectTerminal(creds[4]) + } + _, err := time.Parse(yyyymmdd, creds[1]) + if err != nil { + return nil, errHandler.InvalidDateFormat(creds[1]) + } + return &CredentialsScope{ + Access: creds[0], + Date: creds[1], + Region: creds[2], + }, nil +} + func removeSpace(str string) string { var b strings.Builder b.Grow(len(str)) @@ -270,3 +298,23 @@ func removeSpace(str string) string { } return b.String() } + +func SignPostPolicy(base64Policy, yyyymmdd, region, secretKey string) (string, error) { + signingKey := deriveSigningKey(secretKey, yyyymmdd, region) + sig := hmacSHA256(signingKey, []byte(base64Policy)) + return hex.EncodeToString(sig), nil +} + +func deriveSigningKey(secretKey, yyyymmdd, region string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secretKey), []byte(yyyymmdd)) + kRegion := hmacSHA256(kDate, []byte(region)) + kService := hmacSHA256(kRegion, []byte(service)) + kSigning := hmacSHA256(kService, []byte("aws4_request")) + return kSigning +} + +func hmacSHA256(key, data []byte) []byte { + h := hmac.New(sha256.New, key) + h.Write(data) + return h.Sum(nil) +} diff --git a/s3api/utils/context-keys.go b/s3api/utils/context-keys.go index 76dfde0b..d3503adc 100644 --- a/s3api/utils/context-keys.go +++ b/s3api/utils/context-keys.go @@ -24,19 +24,20 @@ import ( type ContextKey string const ( - ContextKeyRegion ContextKey = "region" - ContextKeyStartTime ContextKey = "start-time" - ContextKeyIsRoot ContextKey = "is-root" - ContextKeyRootAccessKey ContextKey = "root-access-key" - ContextKeyAccount ContextKey = "account" - ContextKeyAuthenticated ContextKey = "authenticated" - ContextKeyPublicBucket ContextKey = "public-bucket" - ContextKeyParsedAcl ContextKey = "parsed-acl" - ContextKeySkipResBodyLog ContextKey = "skip-res-body-log" - ContextKeyBodyReader ContextKey = "body-reader" - ContextKeySkip ContextKey = "__skip" - ContextKeyStack ContextKey = "stack" - ContextKeyBucketOwner ContextKey = "bucket-owner" + ContextKeyRegion ContextKey = "region" + ContextKeyStartTime ContextKey = "start-time" + ContextKeyIsRoot ContextKey = "is-root" + ContextKeyRootAccessKey ContextKey = "root-access-key" + ContextKeyAccount ContextKey = "account" + ContextKeyAuthenticated ContextKey = "authenticated" + ContextKeyPublicBucket ContextKey = "public-bucket" + ContextKeyParsedAcl ContextKey = "parsed-acl" + ContextKeySkipResBodyLog ContextKey = "skip-res-body-log" + ContextKeyBodyReader ContextKey = "body-reader" + ContextKeySkip ContextKey = "__skip" + ContextKeyStack ContextKey = "stack" + ContextKeyBucketOwner ContextKey = "bucket-owner" + ContextKeyObjectPostResult ContextKey = "object-post-result" ) func (ck ContextKey) Values() []ContextKey { diff --git a/s3api/utils/multipart_reader.go b/s3api/utils/multipart_reader.go new file mode 100644 index 00000000..abf22847 --- /dev/null +++ b/s3api/utils/multipart_reader.go @@ -0,0 +1,349 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "bufio" + "bytes" + "fmt" + "io" + "mime" + "net/textproto" + "strings" + + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3err" +) + +// MultipartParser parses S3 browser-based POST multipart/form-data in a streaming way. +// It buffers regular form fields, but it does not buffer the file part. +type MultipartParser struct { + br *bufio.Reader + boundary string + requestContentLength int64 + bytesRead int64 +} + +// NewMultipartParser creates a new streaming multipart parser. +// boundary should be the raw boundary value from Content-Type, without the leading "--". +// If accidentally "--" has been passed, it is normalized. +func NewMultipartParser(body io.Reader, boundary string, requestContentLength int64) (*MultipartParser, error) { + if body == nil { + debuglogger.Logf("multipart parser requires non-nil body reader") + return nil, fmt.Errorf("nil body reader") + } + if requestContentLength < 0 { + debuglogger.Logf("invalid multipart request content-length: %d", requestContentLength) + return nil, fmt.Errorf("invalid request content-length: %d", requestContentLength) + } + + boundary = strings.TrimSpace(boundary) + boundary = strings.TrimPrefix(boundary, "--") + if boundary == "" { + debuglogger.Logf("multipart boundary is empty") + return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + return &MultipartParser{ + br: bufio.NewReader(body), + boundary: boundary, + requestContentLength: requestContentLength, + }, nil +} + +// MpFileReader is the streaming interface for the file part of a multipart POST. +// It extends io.Reader with a Length method that returns the number of file-content +// bytes actually delivered to callers (boundary and delimiter bytes are not counted). +type MpFileReader interface { + io.Reader + Length() int64 +} + +type MpParseResult struct { + // Fields contains all non-file form fields collected before the file part. + Fields map[string]string + // FileRdr streams the file payload without buffering the entire part in memory. + FileRdr MpFileReader + // ContentLength is the expected byte length of the file payload only. + ContentLength int64 +} + +// Parse parses all non-file fields and returns: +// - form values +// - a streaming file reader +// - file content length +// +// The returned file reader MUST be read until EOF, otherwise final-boundary +// validation is not triggered. +func (mp *MultipartParser) Parse() (*MpParseResult, error) { + fields := make(map[string]string) + + if err := mp.expectInitialBoundary(); err != nil { + return nil, err + } + + for { + headers, err := mp.readHeaders() + if err != nil { + return nil, err + } + + cd, ok := headers[textproto.CanonicalMIMEHeaderKey("Content-Disposition")] + if !ok { + debuglogger.Logf("multipart part is missing Content-Disposition header") + return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + disp, params, err := mime.ParseMediaType(cd) + if err != nil { + debuglogger.Logf("invalid multipart Content-Disposition header %q: %v", cd, err) + return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + if disp != "form-data" { + debuglogger.Logf("unexpected multipart disposition: %s", disp) + return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + name := strings.ToLower(params["name"]) + if name == "" { + debuglogger.Logf("multipart part is missing field name") + return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + _, hasFilename := params["filename"] + isFilePart := name == "file" || hasFilename + + // At this point, headers + blank line have already been consumed, + // so bytesRead points exactly at the first byte of file content. + if isFilePart { + fileContentLength := mp.requestContentLength - mp.bytesRead - int64(len(mp.boundary)) - 8 + if fileContentLength < 0 { + debuglogger.Logf("calculated negative multipart file content-length: %d", fileContentLength) + return nil, fmt.Errorf("calculated negative file content-length: %d", fileContentLength) + } + + fr := &finalFileReader{ + r: mp.br, + trailer: []byte("\r\n--" + mp.boundary + "--\r\n"), + } + + return &MpParseResult{ + Fields: fields, + FileRdr: fr, + ContentLength: fileContentLength, + }, nil + } + + value, err := mp.readFieldValue() + if err != nil { + return nil, err + } + + if strings.HasPrefix(name, "x-amz-meta-") { + val, ok := fields[name] + if ok { + fields[name] = val + "," + value + continue + } + } + + fields[name] = value + } +} + +func (mp *MultipartParser) expectInitialBoundary() error { + line, _, err := mp.readLine() + if err != nil { + return s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + want := "--" + mp.boundary + if line != want { + debuglogger.Logf("unexpected initial multipart boundary: got %q want %q", line, want) + return s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + return nil +} + +func (mp *MultipartParser) readHeaders() (map[string]string, error) { + headers := make(map[string]string) + + for { + line, _, err := mp.readLine() + if err != nil { + return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + // Blank line terminates headers. + if line == "" { + return headers, nil + } + + key, value, ok := strings.Cut(line, ":") + if !ok { + debuglogger.Logf("invalid multipart header line: %q", line) + return nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + key = textproto.CanonicalMIMEHeaderKey(strings.TrimSpace(key)) + value = strings.TrimSpace(value) + headers[key] = value + } +} + +// readFieldValue reads a regular form field until the next boundary line. +// It keeps exact field bytes except for the final CRLF that belongs to the boundary separator. +func (mp *MultipartParser) readFieldValue() (string, error) { + boundaryLine := "--" + mp.boundary + finalBoundaryLine := boundaryLine + "--" + + var buf bytes.Buffer + + for { + line, raw, err := mp.readLine() + if err != nil { + return "", err + } + + switch line { + case boundaryLine: + trimTrailingCRLF(&buf) + return buf.String(), nil + + case finalBoundaryLine: + debuglogger.Logf("multipart POST ended before file part was found") + return "", s3err.GetAPIError(s3err.ErrPOSTFileRequired) + + default: + buf.Write(raw) + } + } +} + +// readLine reads one CRLF-terminated line, counts consumed bytes, +// returns the line without trailing CRLF, and also the raw bytes including CRLF. +func (mp *MultipartParser) readLine() (string, []byte, error) { + s, err := mp.br.ReadString('\n') + if err != nil { + debuglogger.Logf("failed to read multipart line: %v", err) + return "", nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + mp.bytesRead += int64(len(s)) + + if !strings.HasSuffix(s, "\r\n") { + debuglogger.Logf("multipart line is not CRLF-terminated: %q", s) + return "", nil, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + } + + return strings.TrimSuffix(s, "\r\n"), []byte(s), nil +} + +func trimTrailingCRLF(buf *bytes.Buffer) { + b := buf.Bytes() + if len(b) >= 2 && b[len(b)-2] == '\r' && b[len(b)-1] == '\n' { + buf.Truncate(len(b) - 2) + } +} + +// finalFileReader streams file bytes until it reaches the final multipart boundary: +// +// \r\n----\r\n +// +// Any epilogue bytes after that final boundary are ignored. +type finalFileReader struct { + r *bufio.Reader + // trailer is the exact byte sequence that terminates the file part. + trailer []byte + // buf keeps unread bytes plus a trailer-sized lookbehind window so + // boundary bytes split across reads are not emitted as file content. + buf []byte + // bytesRead counts only the file-content bytes delivered to callers. + // Boundary and delimiter bytes are never included in this count. + bytesRead int64 + + done bool + failed error + eof bool +} + +func (r *finalFileReader) Read(p []byte) (int, error) { + if r.failed != nil { + return 0, r.failed + } + if r.done { + return 0, io.EOF + } + if len(p) == 0 { + return 0, nil + } + + for { + // If the final boundary is already in the buffer, return only file + // bytes before it and stop cleanly once the boundary starts at buf[0]. + if idx := bytes.Index(r.buf, r.trailer); idx >= 0 { + if idx == 0 { + r.buf = nil + r.done = true + return 0, io.EOF + } + + n := copy(p, r.buf[:idx]) + r.buf = r.buf[n:] + r.bytesRead += int64(n) + return n, nil + } + + // Bytes before this point cannot be part of a future trailer match, so + // they are safe to release to the caller. + safe := len(r.buf) - len(r.trailer) + 1 + if safe > 0 { + n := copy(p, r.buf[:safe]) + r.buf = r.buf[n:] + r.bytesRead += int64(n) + return n, nil + } + + if r.eof { + // Reaching EOF without finding the expected closing boundary means + // the multipart body was truncated or malformed. + debuglogger.Logf("multipart file stream ended before final boundary %q", string(r.trailer)) + r.failed = s3err.GetAPIError(s3err.ErrMalformedPOSTRequest) + return 0, r.failed + } + + chunk := make([]byte, 4096) + n, err := r.r.Read(chunk) + if n > 0 { + r.buf = append(r.buf, chunk[:n]...) + } + + if err == io.EOF { + r.eof = true + continue + } + if err != nil { + debuglogger.Logf("failed to read multipart file data: %v", err) + r.failed = err + return 0, err + } + } +} + +// Length returns the total number of file-content bytes delivered to callers so far. +// Multipart boundary and delimiter bytes are never counted. +func (r *finalFileReader) Length() int64 { + return r.bytesRead +} diff --git a/s3api/utils/multipart_reader_test.go b/s3api/utils/multipart_reader_test.go new file mode 100644 index 00000000..60391203 --- /dev/null +++ b/s3api/utils/multipart_reader_test.go @@ -0,0 +1,514 @@ +package utils + +import ( + "bufio" + "errors" + "io" + "strings" + "testing" + + "github.com/versity/versitygw/s3err" +) + +type chunkedReader struct { + data []byte + steps []int + pos int + idx int +} + +func (r *chunkedReader) Read(p []byte) (int, error) { + if r.pos >= len(r.data) { + return 0, io.EOF + } + + n := len(p) + if r.idx < len(r.steps) && r.steps[r.idx] < n { + n = r.steps[r.idx] + } + remaining := len(r.data) - r.pos + if remaining < n { + n = remaining + } + + copy(p, r.data[r.pos:r.pos+n]) + r.pos += n + r.idx++ + return n, nil +} + +func newMultipartParserForTest(t *testing.T, body, boundary string) *MultipartParser { + t.Helper() + + mp, err := NewMultipartParser(strings.NewReader(body), boundary, int64(len(body))) + if err != nil { + t.Fatalf("new multipart parser: %v", err) + } + + return mp +} + +func TestNewMultipartParserValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body io.Reader + boundary string + length int64 + wantError error + }{ + { + name: "nil body", + boundary: "abc", + length: 1, + wantError: errors.New("nil body reader"), + }, + { + name: "negative content length", + body: strings.NewReader("x"), + boundary: "abc", + length: -1, + wantError: errors.New("invalid request content-length: -1"), + }, + { + name: "empty boundary", + body: strings.NewReader("x"), + boundary: " ", + length: 1, + wantError: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := NewMultipartParser(tt.body, tt.boundary, tt.length) + if err == nil { + t.Fatal("expected error") + } + if err.Error() != tt.wantError.Error() { + t.Fatalf("unexpected error: got %v want %v", err, tt.wantError) + } + }) + } +} + +func TestMultipartParserParseSuccess(t *testing.T) { + t.Parallel() + + body := strings.Join([]string{ + "--abc\r\n", + "Content-Disposition: form-data; name=\"key\"\r\n", + "\r\n", + "uploads/photo.jpg\r\n", + "--abc\r\n", + "Content-Disposition: form-data; name=\"success_action_status\"\r\n", + "\r\n", + "201\r\n", + "--abc\r\n", + "Content-Disposition: form-data; name=\"x-amz-meta-color\"\r\n", + "\r\n", + "blue\r\n", + "--abc\r\n", + "Content-Disposition: form-data; name=\"x-amz-meta-color\"\r\n", + "\r\n", + "green\r\n", + "--abc\r\n", + "Content-Disposition: form-data; name=\"file\"; filename=\"photo.jpg\"\r\n", + "Content-Type: image/jpeg\r\n", + "\r\n", + "file-body-123", + "\r\n--abc--\r\n", + }, "") + + mp := newMultipartParserForTest(t, body, "--abc") + + got, err := mp.Parse() + if err != nil { + t.Fatalf("parse: %v", err) + } + + if got.Fields["key"] != "uploads/photo.jpg" { + t.Fatalf("unexpected key field: %q", got.Fields["key"]) + } + if got.Fields["success_action_status"] != "201" { + t.Fatalf("unexpected status field: %q", got.Fields["success_action_status"]) + } + if got.Fields["x-amz-meta-color"] != "blue,green" { + t.Fatalf("unexpected merged metadata field: %q", got.Fields["x-amz-meta-color"]) + } + if got.ContentLength != int64(len("file-body-123")) { + t.Fatalf("unexpected file content-length: got %d want %d", got.ContentLength, len("file-body-123")) + } + + fileData, err := io.ReadAll(got.FileRdr) + if err != nil { + t.Fatalf("read file: %v", err) + } + if string(fileData) != "file-body-123" { + t.Fatalf("unexpected file data: %q", fileData) + } + + n, err := got.FileRdr.Read(make([]byte, 1)) + if n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("expected EOF after file reader drained, got n=%d err=%v", n, err) + } +} + +func TestMultipartParserParsePreservesMultilineFieldValue(t *testing.T) { + t.Parallel() + + body := strings.Join([]string{ + "--abc\r\n", + "Content-Disposition: form-data; name=\"policy\"\r\n", + "\r\n", + "line-one\r\n", + "line-two\r\n", + "--abc\r\n", + "Content-Disposition: form-data; name=\"file\"; filename=\"note.txt\"\r\n", + "\r\n", + "payload", + "\r\n--abc--\r\n", + }, "") + + mp := newMultipartParserForTest(t, body, "abc") + + got, err := mp.Parse() + if err != nil { + t.Fatalf("parse: %v", err) + } + + if got.Fields["policy"] != "line-one\r\nline-two" { + t.Fatalf("unexpected multiline field: %q", got.Fields["policy"]) + } +} + +func TestMultipartParserRecognizesFilenameOnlyFilePart(t *testing.T) { + t.Parallel() + + body := strings.Join([]string{ + "--abc\r\n", + "Content-Disposition: form-data; name=\"key\"\r\n", + "\r\n", + "value\r\n", + "--abc\r\n", + "Content-Disposition: form-data; name=\"upload\"; filename=\"blob.bin\"\r\n", + "\r\n", + "xyz", + "\r\n--abc--\r\n", + }, "") + + mp := newMultipartParserForTest(t, body, "abc") + + got, err := mp.Parse() + if err != nil { + t.Fatalf("parse: %v", err) + } + if got.ContentLength != 3 { + t.Fatalf("unexpected file content-length: got %d want 3", got.ContentLength) + } + + fileData, err := io.ReadAll(got.FileRdr) + if err != nil { + t.Fatalf("read file: %v", err) + } + if string(fileData) != "xyz" { + t.Fatalf("unexpected file data: %q", fileData) + } +} + +func TestMultipartParserParseErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + want error + }{ + { + name: "invalid initial boundary", + body: strings.Join([]string{ + "--wrong\r\n", + "Content-Disposition: form-data; name=\"file\"; filename=\"x\"\r\n", + "\r\n", + "payload\r\n", + "--abc--\r\n", + }, ""), + want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest), + }, + { + name: "missing content disposition header", + body: strings.Join([]string{ + "--abc\r\n", + "Content-Type: text/plain\r\n", + "\r\n", + "value\r\n", + "--abc--\r\n", + }, ""), + want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest), + }, + { + name: "invalid header format", + body: strings.Join([]string{ + "--abc\r\n", + "Content-Disposition form-data; name=\"key\"\r\n", + "\r\n", + "value\r\n", + "--abc--\r\n", + }, ""), + want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest), + }, + { + name: "invalid content disposition media type", + body: strings.Join([]string{ + "--abc\r\n", + "Content-Disposition: attachment; name=\"key\"\r\n", + "\r\n", + "value\r\n", + "--abc--\r\n", + }, ""), + want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest), + }, + { + name: "missing content disposition name", + body: strings.Join([]string{ + "--abc\r\n", + "Content-Disposition: form-data\r\n", + "\r\n", + "value\r\n", + "--abc--\r\n", + }, ""), + want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest), + }, + { + name: "missing file part", + body: strings.Join([]string{ + "--abc\r\n", + "Content-Disposition: form-data; name=\"key\"\r\n", + "\r\n", + "value\r\n", + "--abc--\r\n", + }, ""), + want: s3err.GetAPIError(s3err.ErrPOSTFileRequired), + }, + { + name: "line without crlf terminator", + body: strings.Join([]string{ + "--abc\n", + "Content-Disposition: form-data; name=\"file\"; filename=\"x\"\r\n", + "\r\n", + "payload\r\n", + "--abc--\r\n", + }, ""), + want: s3err.GetAPIError(s3err.ErrMalformedPOSTRequest), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + mp := newMultipartParserForTest(t, tt.body, "abc") + _, err := mp.Parse() + if !errors.Is(err, tt.want) { + t.Fatalf("unexpected error: got %v want %v", err, tt.want) + } + }) + } +} + +func TestMultipartParserFileReaderRequiresFinalBoundary(t *testing.T) { + t.Parallel() + + body := strings.Join([]string{ + "--abc\r\n", + "Content-Disposition: form-data; name=\"file\"; filename=\"x\"\r\n", + "\r\n", + "payload-without-closing-boundary", + }, "") + + mp := newMultipartParserForTest(t, body, "abc") + + got, err := mp.Parse() + if err != nil { + t.Fatalf("parse: %v", err) + } + + _, err = io.ReadAll(got.FileRdr) + if !errors.Is(err, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)) { + t.Fatalf("expected malformed post request, got %v", err) + } +} + +func TestFinalFileReaderStopsAtFinalBoundary(t *testing.T) { + t.Parallel() + + r := &finalFileReader{ + r: bufio.NewReader(strings.NewReader("hello world\r\n--abc--\r\ntrailing-data")), + trailer: []byte("\r\n--abc--\r\n"), + } + + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read all: %v", err) + } + + if string(got) != "hello world" { + t.Fatalf("unexpected body: got %q", got) + } + + n, err := r.Read(make([]byte, 8)) + if n != 0 { + t.Fatalf("expected zero bytes after EOF, got %d", n) + } + if !errors.Is(err, io.EOF) { + t.Fatalf("expected EOF after final boundary, got %v", err) + } +} + +func TestFinalFileReaderHandlesBoundarySplitAcrossReads(t *testing.T) { + t.Parallel() + + src := &chunkedReader{ + data: []byte("hello world\r\n--abc--\r\nignored"), + steps: []int{5, 3, 2, 1, 4, 2, 3, 10}, + } + r := &finalFileReader{ + r: bufio.NewReader(src), + trailer: []byte("\r\n--abc--\r\n"), + } + + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read all: %v", err) + } + + if string(got) != "hello world" { + t.Fatalf("unexpected body: got %q", got) + } +} + +func TestFinalFileReaderMissingFinalBoundary(t *testing.T) { + t.Parallel() + + r := &finalFileReader{ + r: bufio.NewReader(strings.NewReader("hello world")), + trailer: []byte("\r\n--abc--\r\n"), + } + + _, err := io.ReadAll(r) + if !errors.Is(err, s3err.GetAPIError(s3err.ErrMalformedPOSTRequest)) { + t.Fatalf("expected malformed post request, got %v", err) + } +} + +func TestFinalFileReaderLengthCountsOnlyFileBytes(t *testing.T) { + t.Parallel() + + fileContent := "hello world" + // trailer bytes must not be included in Length() + src := fileContent + "\r\n--abc--\r\n" + + r := &finalFileReader{ + r: bufio.NewReader(strings.NewReader(src)), + trailer: []byte("\r\n--abc--\r\n"), + } + + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read all: %v", err) + } + if string(got) != fileContent { + t.Fatalf("unexpected body: got %q want %q", got, fileContent) + } + if r.Length() != int64(len(fileContent)) { + t.Fatalf("unexpected length: got %d want %d", r.Length(), len(fileContent)) + } +} + +func TestFinalFileReaderLengthZeroBeforeRead(t *testing.T) { + t.Parallel() + + r := &finalFileReader{ + r: bufio.NewReader(strings.NewReader("data\r\n--b--\r\n")), + trailer: []byte("\r\n--b--\r\n"), + } + + if r.Length() != 0 { + t.Fatalf("expected length 0 before any reads, got %d", r.Length()) + } +} + +func TestFinalFileReaderLengthIncrementalReads(t *testing.T) { + t.Parallel() + + fileContent := "abcdefgh" + src := &chunkedReader{ + data: []byte(fileContent + "\r\n--abc--\r\n"), + steps: []int{3, 3, 2, 10}, + } + r := &finalFileReader{ + r: bufio.NewReader(src), + trailer: []byte("\r\n--abc--\r\n"), + } + + var total int + buf := make([]byte, 4) + for { + n, err := r.Read(buf) + total += n + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("unexpected read error: %v", err) + } + } + + if r.Length() != int64(len(fileContent)) { + t.Fatalf("length after incremental reads: got %d want %d", r.Length(), len(fileContent)) + } + if int64(total) != r.Length() { + t.Fatalf("total bytes returned by Read (%d) does not match Length() (%d)", total, r.Length()) + } +} + +func TestMultipartParserFileReaderLength(t *testing.T) { + t.Parallel() + + fileContent := "file-body-123" + body := strings.Join([]string{ + "--abc\r\n", + "Content-Disposition: form-data; name=\"key\"\r\n", + "\r\n", + "mykey\r\n", + "--abc\r\n", + "Content-Disposition: form-data; name=\"file\"; filename=\"f.bin\"\r\n", + "\r\n", + fileContent, + "\r\n--abc--\r\n", + }, "") + + mp := newMultipartParserForTest(t, body, "abc") + + got, err := mp.Parse() + if err != nil { + t.Fatalf("parse: %v", err) + } + + if got.FileRdr.Length() != 0 { + t.Fatalf("expected Length 0 before reading, got %d", got.FileRdr.Length()) + } + + if _, err := io.ReadAll(got.FileRdr); err != nil { + t.Fatalf("read all: %v", err) + } + + if got.FileRdr.Length() != int64(len(fileContent)) { + t.Fatalf("unexpected Length after read: got %d want %d", got.FileRdr.Length(), len(fileContent)) + } +} diff --git a/s3api/utils/presign-auth-reader.go b/s3api/utils/presign-auth-reader.go index e4328cf1..a545be7e 100644 --- a/s3api/utils/presign-auth-reader.go +++ b/s3api/utils/presign-auth-reader.go @@ -133,7 +133,7 @@ func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, strea // &X-Amz-Expires=86400 // &X-Amz-SignedHeaders=host // &X-Amz-Signature=1e68ad45c1db540284a4a1eca3884c293ba1a0ff63ab9db9a15b5b29dfa02cd8 -func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) { +func ParsePresignedURIParts(ctx *fiber.Ctx, region string) (AuthData, error) { a := AuthData{} // Get and verify algorithm query parameter @@ -149,34 +149,14 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) { return a, s3err.QueryAuthErrors.MissingRequiredParams() } - creds := strings.Split(credsQuery, "/") - if len(creds) != 5 { - return a, s3err.QueryAuthErrors.MalformedCredential() - } - - // validate the service - if creds[3] != "s3" { - return a, s3err.QueryAuthErrors.IncorrectService(creds[3]) - } - - // validate the terminal - if creds[4] != "aws4_request" { - return a, s3err.QueryAuthErrors.IncorrectTerminal(creds[4]) - } - - // validate the date - _, err = time.Parse(yyyymmdd, creds[1]) + creds, err := ParseCredentials(credsQuery, s3err.QueryAuthErrors) if err != nil { - return a, s3err.QueryAuthErrors.InvalidDateFormat(creds[1]) + return a, err } - region, ok := ContextKeyRegion.Get(ctx).(string) - if !ok { - region = "" - } // validate the region - if creds[2] != region { - return a, s3err.QueryAuthErrors.IncorrectRegion(region, creds[2]) + if creds.Region != region { + return a, s3err.QueryAuthErrors.IncorrectRegion(region, creds.Region) } // Parse and validate Date query param @@ -190,8 +170,8 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) { return a, s3err.QueryAuthErrors.InvalidXAmzDateFormat() } - if date[:8] != creds[1] { - return a, s3err.QueryAuthErrors.DateMismatch(creds[1], date[:8]) + if date[:8] != creds.Date { + return a, s3err.QueryAuthErrors.DateMismatch(creds.Date, date[:8]) } signature := ctx.Query("X-Amz-Signature") @@ -211,9 +191,9 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) { } a.Signature = signature - a.Access = creds[0] + a.Access = creds.Access a.Algorithm = algo - a.Region = creds[2] + a.Region = creds.Region a.SignedHeaders = signedHdrs a.Date = date diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index c87fad9a..90cc545f 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -103,6 +103,28 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (map[string]string, error) return metadata, nil } +func ExtractMetadataFromFields(fields map[string]string) (map[string]string, error) { + metadata := make(map[string]string) + var metadataSize int + + for key, value := range fields { + if !strings.HasPrefix(key, "x-amz-meta-") { + continue + } + + trimmedKey := key[11:] + metadataSize += len(trimmedKey) + len(value) + if metadataSize > maxMetadataSize { + debuglogger.Logf("total meta headers size exceeded the maximum allowed: (size): %v, (max): %v", metadataSize, maxMetadataSize) + return nil, s3err.GetAPIError(s3err.ErrMetadataTooLarge) + } + + metadata[trimmedKey] = value + } + + return metadata, nil +} + func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64, streamBody bool) (*http.Request, error) { req := ctx.Request() var body io.Reader @@ -554,6 +576,53 @@ func ParseCalculatedChecksumHeaders(ctx *fiber.Ctx) (ChecksumValues, error) { return checksums, nil } +// ParseCalculatedChecksumFields parses and validates object POST checksum fields +func ParseCalculatedChecksumFields(fields map[string]string) (ChecksumValues, error) { + checksums := ChecksumValues{} + + var hdrErr error + // Parse and validate checksum headers + for key, value := range fields { + // only check the headers with 'X-Amz-Checksum-' prefix + if !strings.HasPrefix(key, "x-amz-checksum-") { + continue + } + // "x-amz-checksum-type" and "x-amz-checksum-algorithm" aren't considered + // as invalid values, even if the s3 action doesn't expect these headers + switch key { + case "x-amz-checksum-type", "x-amz-checksum-algorithm": + continue + } + + algo := types.ChecksumAlgorithm(strings.ToUpper(strings.TrimPrefix(key, "x-amz-checksum-"))) + err := IsChecksumAlgorithmValid(algo) + if err != nil { + debuglogger.Logf("invalid checksum field: %s\n", key) + hdrErr = s3err.GetAPIError(s3err.ErrInvalidChecksumHeader) + break + } + + checksums[algo] = value + } + + if hdrErr != nil { + return checksums, hdrErr + } + + if len(checksums) > 1 { + debuglogger.Logf("multiple checksum headers provided: %v\n", checksums.Headers()) + return checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders) + } + + for al, val := range checksums { + if !IsValidChecksum(val, al) { + return checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al)))) + } + } + + return checksums, nil +} + // ParseCompleteMpChecksumHeaders parses and validates // the 'CompleteMultipartUpload' x-amz-checksum-x headers // by supporting both 'checksum' and 'checksum-' formats @@ -805,7 +874,7 @@ var tagRule = regexp.MustCompile(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`) // Parses and validates tagging func ParseTagging(data []byte, limit TagLimit) (map[string]string, error) { - var tagging s3response.TaggingInput + var tagging s3response.Tagging err := xml.Unmarshal(data, &tagging) if err != nil { debuglogger.Logf("invalid taggging: %s", data) @@ -864,6 +933,22 @@ func ParseTagging(data []byte, limit TagLimit) (map[string]string, error) { return tagSet, nil } +// ConvertTaggingXMLToQueryString parses object tagging XML and converts it into the +// x-amz-tagging query-string form expected by PutObject requests. +func ConvertTaggingXMLToQueryString(data []byte) (string, error) { + tags, err := ParseTagging(data, TagLimitObject) + if err != nil { + return "", err + } + + values := url.Values{} + for key, value := range tags { + values.Set(key, value) + } + + return values.Encode(), nil +} + // Returns the provided string pointer func GetStringPtr(str string) *string { if str == "" { diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 0538bb2c..218a5163 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -21,6 +21,7 @@ import ( "errors" "math/rand" "net/http" + "net/url" "reflect" "strings" "testing" @@ -815,6 +816,142 @@ func TestIsValidChecksum(t *testing.T) { } } +func TestExtractMetadataFromFields(t *testing.T) { + tests := []struct { + name string + fields map[string]string + want map[string]string + wantErr error + }{ + { + name: "extracts metadata only", + fields: map[string]string{ + "x-amz-meta-owner": "alice", + "x-amz-meta-env": "prod", + "key": "uploads/report.pdf", + }, + want: map[string]string{ + "owner": "alice", + "env": "prod", + }, + }, + { + name: "returns empty map when no metadata fields exist", + fields: map[string]string{ + "key": "uploads/report.pdf", + "acl": "private", + "file": "ignored", + }, + want: map[string]string{}, + }, + { + name: "allows empty metadata value", + fields: map[string]string{ + "x-amz-meta-owner": "", + }, + want: map[string]string{ + "owner": "", + }, + }, + { + name: "metadata too large", + fields: map[string]string{ + "x-amz-meta-big": strings.Repeat("a", maxMetadataSize-len("big")+1), + }, + wantErr: s3err.GetAPIError(s3err.ErrMetadataTooLarge), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ExtractMetadataFromFields(tt.fields) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected error %v, got %v", tt.wantErr, err) + } + if err != nil { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("unexpected metadata: got %v want %v", got, tt.want) + } + }) + } +} + +func TestParseCalculatedChecksumFields(t *testing.T) { + tests := []struct { + name string + fields map[string]string + want ChecksumValues + wantErr error + }{ + { + name: "empty fields", + fields: map[string]string{ + "key": "uploads/file.txt", + }, + want: ChecksumValues{}, + }, + { + name: "single valid checksum field", + fields: map[string]string{ + "x-amz-checksum-crc32": "ww2FVQ==", + }, + want: ChecksumValues{ + types.ChecksumAlgorithmCrc32: "ww2FVQ==", + }, + }, + { + name: "ignores algorithm and type helper fields", + fields: map[string]string{ + "x-amz-checksum-algorithm": "CRC32", + "x-amz-checksum-type": "FULL_OBJECT", + "x-amz-checksum-crc32": "ww2FVQ==", + }, + want: ChecksumValues{ + types.ChecksumAlgorithmCrc32: "ww2FVQ==", + }, + }, + { + name: "invalid checksum field name", + fields: map[string]string{ + "x-amz-checksum-madeup": "abc", + }, + wantErr: s3err.GetAPIError(s3err.ErrInvalidChecksumHeader), + }, + { + name: "multiple checksum fields", + fields: map[string]string{ + "x-amz-checksum-crc32": "ww2FVQ==", + "x-amz-checksum-sha256": "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=", + }, + wantErr: s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders), + }, + { + name: "invalid checksum value", + fields: map[string]string{ + "x-amz-checksum-crc32": "invalid_base64_string", + }, + wantErr: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseCalculatedChecksumFields(tt.fields) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected error %v, got %v", tt.wantErr, err) + } + if err != nil { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("unexpected checksums: got %v want %v", got, tt.want) + } + }) + } +} + func TestIsChecksumTypeValid(t *testing.T) { type args struct { t types.ChecksumType @@ -985,8 +1122,8 @@ func TestParseTagging(t *testing.T) { } return string(b) } - getTagSet := func(lgth int) s3response.TaggingInput { - res := s3response.TaggingInput{ + getTagSet := func(lgth int) s3response.Tagging { + res := s3response.Tagging{ TagSet: s3response.TagSet{ Tags: []s3response.Tag{}, }, @@ -1002,7 +1139,7 @@ func TestParseTagging(t *testing.T) { return res } type args struct { - data s3response.TaggingInput + data s3response.Tagging overrideXML []byte limit TagLimit } @@ -1015,7 +1152,7 @@ func TestParseTagging(t *testing.T) { { name: "valid tags within limit", args: args{ - data: s3response.TaggingInput{ + data: s3response.Tagging{ TagSet: s3response.TagSet{ Tags: []s3response.Tag{ {Key: "key1", Value: "value1"}, @@ -1037,6 +1174,24 @@ func TestParseTagging(t *testing.T) { want: nil, wantErr: s3err.GetAPIError(s3err.ErrMalformedXML), }, + { + name: "valid tags without namespace", + args: args{ + overrideXML: []byte(`key1value1key2value2`), + limit: TagLimitObject, + }, + want: map[string]string{"key1": "value1", "key2": "value2"}, + wantErr: nil, + }, + { + name: "valid tags with namespace", + args: args{ + overrideXML: []byte(`key1value1key2value2`), + limit: TagLimitObject, + }, + want: map[string]string{"key1": "value1", "key2": "value2"}, + wantErr: nil, + }, { name: "exceeds bucket tag limit", args: args{ @@ -1058,7 +1213,7 @@ func TestParseTagging(t *testing.T) { { name: "invalid 0 length tag key", args: args{ - data: s3response.TaggingInput{ + data: s3response.Tagging{ TagSet: s3response.TagSet{ Tags: []s3response.Tag{{Key: "", Value: "value1"}}, }, @@ -1071,7 +1226,7 @@ func TestParseTagging(t *testing.T) { { name: "invalid long tag key", args: args{ - data: s3response.TaggingInput{ + data: s3response.Tagging{ TagSet: s3response.TagSet{ Tags: []s3response.Tag{{Key: genRandStr(130), Value: "value1"}}, }, @@ -1084,7 +1239,7 @@ func TestParseTagging(t *testing.T) { { name: "invalid long tag value", args: args{ - data: s3response.TaggingInput{ + data: s3response.Tagging{ TagSet: s3response.TagSet{ Tags: []s3response.Tag{{Key: "key", Value: genRandStr(257)}}, }, @@ -1097,7 +1252,7 @@ func TestParseTagging(t *testing.T) { { name: "duplicate tag key", args: args{ - data: s3response.TaggingInput{ + data: s3response.Tagging{ TagSet: s3response.TagSet{ Tags: []s3response.Tag{ {Key: "key", Value: "value1"}, @@ -1136,6 +1291,71 @@ func TestParseTagging(t *testing.T) { } } +func TestConvertTaggingXMLToQueryString(t *testing.T) { + tests := []struct { + name string + data s3response.Tagging + rawXML []byte + want string + wantErr error + }{ + { + name: "success", + data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + {Key: "project", Value: "versity gw"}, + {Key: "team", Value: "storage"}, + }, + }, + }, + want: "project=versity+gw&team=storage", + }, + { + name: "parse error", + rawXML: []byte("not xml"), + wantErr: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var data []byte + if tt.rawXML != nil { + data = tt.rawXML + } else { + var err error + data, err = xml.Marshal(tt.data) + if err != nil { + t.Fatalf("error marshalling input: %v", err) + } + } + + got, err := ConvertTaggingXMLToQueryString(data) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected error %v, got %v", tt.wantErr, err) + } + if err != nil { + return + } + + if got != tt.want { + t.Fatalf("unexpected tagging string: got %q want %q", got, tt.want) + } + + values, err := url.ParseQuery(got) + if err != nil { + t.Fatalf("parse query: %v", err) + } + for _, tag := range tt.data.TagSet.Tags { + if values.Get(tag.Key) != tag.Value { + t.Fatalf("unexpected query value for %q: got %q want %q", tag.Key, values.Get(tag.Key), tag.Value) + } + } + }) + } +} + func TestValidateCopySource(t *testing.T) { tests := []struct { name string diff --git a/s3err/post-object.go b/s3err/post-object.go new file mode 100644 index 00000000..7696b17e --- /dev/null +++ b/s3err/post-object.go @@ -0,0 +1,147 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package s3err + +import ( + "fmt" + "net/http" +) + +// Factory for building s3 object POST authentication errors. +func invalidPOSTObjectAuthErr(format string, args ...any) APIError { + return APIError{ + Code: "InvalidArgument", + Description: fmt.Sprintf(format, args...), + HTTPStatusCode: http.StatusBadRequest, + } +} + +type invalidPostAuthErr struct{} + +func (invalidPostAuthErr) InvalidDateFormat(s string) APIError { + return invalidPOSTObjectAuthErr( + "incorrect date format %q. This date in the credential must be in the format \"yyyyMMdd\".", + s, + ) +} + +func (invalidPostAuthErr) MalformedCredential() APIError { + return invalidPOSTObjectAuthErr( + "the Credential is mal-formed; expecting \"/YYYYMMDD/REGION/SERVICE/aws4_request\".", + ) +} + +func (invalidPostAuthErr) IncorrectTerminal(s string) APIError { + return invalidPOSTObjectAuthErr("incorrect terminal %q. This endpoint uses \"aws4_request\".", s) +} + +func (invalidPostAuthErr) IncorrectRegion(expected, actual string) APIError { + return invalidPOSTObjectAuthErr("the region %q is wrong; expecting %q", actual, expected) +} + +func (invalidPostAuthErr) IncorrectService(s string) APIError { + return invalidPOSTObjectAuthErr("incorrect service %q. This endpoint belongs to \"s3\".", s) +} + +func (invalidPostAuthErr) MissingField(field string) APIError { + return invalidPOSTObjectAuthErr("Bucket POST must contain a field named '%s'. If it is specified, please check the order of the fields.", field) +} + +var PostAuth invalidPostAuthErr + +// Factory for building s3 object POST authentication errors. +func invalidPolicyDocumentErr(format string, args ...any) APIError { + return APIError{ + Code: "InvalidPolicyDocument", + Description: fmt.Sprintf(format, args...), + HTTPStatusCode: http.StatusBadRequest, + } +} + +func invalidAccordingToPolicyErr(format string, args ...any) APIError { + return APIError{ + Code: "AccessDenied", + Description: fmt.Sprintf(format, args...), + HTTPStatusCode: http.StatusForbidden, + } +} + +type invalidPolicyDocument struct{} + +func (invalidPolicyDocument) EmptyPolicy() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Expecting '{' but found End-of-Input") +} + +func (invalidPolicyDocument) InvalidBase64Encoding() APIError { + return invalidPolicyDocumentErr("Invalid Policy: invalid Base64 encoding.") +} + +func (invalidPolicyDocument) InvalidJSON() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid JSON.") +} + +func (invalidPolicyDocument) MissingExpiration() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Policy missing expiration.") +} + +func (invalidPolicyDocument) InvalidExpiration(exp string) APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid 'expiration' value: '%s'", exp) +} + +func (invalidPolicyDocument) InvalidConditions() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid 'conditions' value: must be a List.") +} + +func (invalidPolicyDocument) InvalidCondition() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid condition test: must be a List or Object.") +} + +func (invalidPolicyDocument) MissingConditionOperationIdentifier() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid Condition: missing operation identifier.") +} + +func (invalidPolicyDocument) UnknownConditionOperation(op string) APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid Condition: unknown operation '%s'.", op) +} + +func (invalidPolicyDocument) IncorrectConditionArgumentsNumber(op string) APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid %s: wrong number of arguments.", op) +} + +func (invalidPolicyDocument) MissingConditions() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Policy missing conditions.") +} + +func (invalidPolicyDocument) OnePropSimpleCondition() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid Simple-Condition: Simple-Conditions must have exactly one property specified.") +} + +func (invalidPolicyDocument) InvalidSimpleCondition() APIError { + return invalidPolicyDocumentErr("Invalid Policy: Invalid Simple-Condition: value must be a string.") +} + +func (invalidPolicyDocument) ConditionFailed(condition string) APIError { + return invalidAccordingToPolicyErr("Invalid according to Policy: Policy Condition failed: %s", condition) +} + +func (invalidPolicyDocument) ExtraInputField(field string) APIError { + return invalidAccordingToPolicyErr("Invalid according to Policy: Extra input fields: %s", field) +} + +func (invalidPolicyDocument) PolicyExpired() APIError { + return invalidAccordingToPolicyErr("Invalid according to Policy: Policy expired.") +} + +var InvalidPolicyDocument invalidPolicyDocument diff --git a/s3err/presigned-urls.go b/s3err/presigned-urls.go index 788d01b0..7a65a5d0 100644 --- a/s3err/presigned-urls.go +++ b/s3err/presigned-urls.go @@ -28,68 +28,63 @@ func authQueryParamError(format string, args ...any) APIError { } } -var QueryAuthErrors = struct { - UnsupportedAlgorithm func() APIError - MalformedCredential func() APIError - IncorrectService func(string) APIError - IncorrectRegion func(expected, actual string) APIError - IncorrectTerminal func(string) APIError - InvalidDateFormat func(string) APIError - DateMismatch func(expected, actual string) APIError - ExpiresTooLarge func() APIError - ExpiresNegative func() APIError - ExpiresNumber func() APIError - MissingRequiredParams func() APIError - InvalidXAmzDateFormat func() APIError - RequestNotYetValid func() APIError - RequestExpired func() APIError - InvalidAccessKeyId func() APIError - // a custom non-AWS error - OnlyHMACSupported func() APIError - SecurityTokenNotSupported func() APIError -}{ +type queryAuthErrors struct{} - UnsupportedAlgorithm: func() APIError { - return authQueryParamError(`X-Amz-Algorithm only supports "AWS4-HMAC-SHA256 and AWS4-ECDSA-P256-SHA256"`) - }, - MalformedCredential: func() APIError { - return authQueryParamError(`Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting "/YYYYMMDD/REGION/SERVICE/aws4_request".`) - }, - IncorrectService: func(s string) APIError { - return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect service %q. This endpoint belongs to "s3".`, s) - }, - IncorrectRegion: func(expected, actual string) APIError { - return authQueryParamError(`Error parsing the X-Amz-Credential parameter; the region %q is wrong; expecting %q`, actual, expected) - }, - IncorrectTerminal: func(s string) APIError { - return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect terminal %q. This endpoint uses "aws4_request".`, s) - }, - InvalidDateFormat: func(s string) APIError { - return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect date format %q. This date in the credential must be in the format "yyyyMMdd".`, s) - }, - DateMismatch: func(expected, actual string) APIError { - return authQueryParamError(`Invalid credential date %q. This date is not the same as X-Amz-Date: %q.`, expected, actual) - }, - ExpiresTooLarge: func() APIError { - return authQueryParamError("X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds") - }, - ExpiresNegative: func() APIError { - return authQueryParamError("X-Amz-Expires must be non-negative") - }, - ExpiresNumber: func() APIError { - return authQueryParamError("X-Amz-Expires should be a number") - }, - MissingRequiredParams: func() APIError { - return authQueryParamError("Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.") - }, - InvalidXAmzDateFormat: func() APIError { - return authQueryParamError(`X-Amz-Date must be in the ISO8601 Long Format "yyyyMMdd'T'HHmmss'Z'"`) - }, - // a custom non-AWS error - OnlyHMACSupported: func() APIError { - return authQueryParamError("X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\"") - }, - SecurityTokenNotSupported: func() APIError { - return authQueryParamError("Authorization with X-Amz-Security-Token is not supported") - }, +func (queryAuthErrors) UnsupportedAlgorithm() APIError { + return authQueryParamError(`X-Amz-Algorithm only supports "AWS4-HMAC-SHA256 and AWS4-ECDSA-P256-SHA256"`) } + +func (queryAuthErrors) MalformedCredential() APIError { + return authQueryParamError(`Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting "/YYYYMMDD/REGION/SERVICE/aws4_request".`) +} + +func (queryAuthErrors) IncorrectService(s string) APIError { + return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect service %q. This endpoint belongs to "s3".`, s) +} + +func (queryAuthErrors) IncorrectRegion(expected, actual string) APIError { + return authQueryParamError(`Error parsing the X-Amz-Credential parameter; the region %q is wrong; expecting %q`, actual, expected) +} + +func (queryAuthErrors) IncorrectTerminal(s string) APIError { + return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect terminal %q. This endpoint uses "aws4_request".`, s) +} + +func (queryAuthErrors) InvalidDateFormat(s string) APIError { + return authQueryParamError(`Error parsing the X-Amz-Credential parameter; incorrect date format %q. This date in the credential must be in the format "yyyyMMdd".`, s) +} + +func (queryAuthErrors) DateMismatch(expected, actual string) APIError { + return authQueryParamError(`Invalid credential date %q. This date is not the same as X-Amz-Date: %q.`, expected, actual) +} + +func (queryAuthErrors) ExpiresTooLarge() APIError { + return authQueryParamError("X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds") +} + +func (queryAuthErrors) ExpiresNegative() APIError { + return authQueryParamError("X-Amz-Expires must be non-negative") +} + +func (queryAuthErrors) ExpiresNumber() APIError { + return authQueryParamError("X-Amz-Expires should be a number") +} + +func (queryAuthErrors) MissingRequiredParams() APIError { + return authQueryParamError("Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.") +} + +func (queryAuthErrors) InvalidXAmzDateFormat() APIError { + return authQueryParamError(`X-Amz-Date must be in the ISO8601 Long Format "yyyyMMdd'T'HHmmss'Z'"`) +} + +// a custom non-AWS error +func (queryAuthErrors) OnlyHMACSupported() APIError { + return authQueryParamError("X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\"") +} + +func (queryAuthErrors) SecurityTokenNotSupported() APIError { + return authQueryParamError("Authorization with X-Amz-Security-Token is not supported") +} + +var QueryAuthErrors queryAuthErrors diff --git a/s3err/s3err.go b/s3err/s3err.go index d40dcc4c..b2659957 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -186,6 +186,8 @@ const ( ErrInvalidChunkSize ErrSlowDown ErrMetadataTooLarge + ErrOnlyAws4HmacSha256 + ErrInvalidDateHeader // Non-AWS errors ErrExistingObjectIsDirectory @@ -439,7 +441,7 @@ var errorCodeResponse = map[ErrorCode]APIError{ }, ErrEntityTooSmall: { Code: "EntityTooSmall", - Description: "Your proposed upload is smaller than the minimum allowed object size.", + Description: "Your proposed upload is smaller than the minimum allowed size", HTTPStatusCode: http.StatusBadRequest, }, ErrEntityTooLarge: { @@ -843,6 +845,16 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Your metadata headers exceed the maximum allowed metadata size", HTTPStatusCode: http.StatusBadRequest, }, + ErrOnlyAws4HmacSha256: { + Code: "InvalidArgument", + Description: "Only AWS4-HMAC-SHA256 is supported", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidDateHeader: { + Code: "InvalidArgument", + Description: "X-Amz-Date must be formated via ISO8601 Long format", + HTTPStatusCode: http.StatusBadRequest, + }, // non aws errors ErrExistingObjectIsDirectory: { diff --git a/s3err/sigv4.go b/s3err/sigv4.go index 5eba1f82..e831a67a 100644 --- a/s3err/sigv4.go +++ b/s3err/sigv4.go @@ -28,50 +28,59 @@ func malformedAuthError(format string, args ...any) APIError { } } -var MalformedAuth = struct { - InvalidDateFormat func(string) APIError - MalformedCredential func() APIError - MissingCredential func() APIError - MissingSignature func() APIError - MissingSignedHeaders func() APIError - InvalidTerminal func(string) APIError - IncorrectRegion func(expected, actual string) APIError - IncorrectService func(string) APIError - MalformedComponent func(string) APIError - MissingComponents func() APIError - DateMismatch func() APIError -}{ - InvalidDateFormat: func(s string) APIError { - return malformedAuthError("incorrect date format %q. This date in the credential must be in the format \"yyyyMMdd\".", s) - }, - MalformedCredential: func() APIError { - return malformedAuthError("the Credential is mal-formed; expecting \"/YYYYMMDD/REGION/SERVICE/aws4_request\".") - }, - MissingCredential: func() APIError { - return malformedAuthError("missing Credential.") - }, - MissingSignature: func() APIError { - return malformedAuthError("missing Signature.") - }, - MissingSignedHeaders: func() APIError { - return malformedAuthError("missing SignedHeaders.") - }, - InvalidTerminal: func(s string) APIError { - return malformedAuthError("incorrect terminal %q. This endpoint uses \"aws4_request\".", s) - }, - IncorrectRegion: func(expected, actual string) APIError { - return malformedAuthError("the region %q is wrong; expecting %q", actual, expected) - }, - IncorrectService: func(s string) APIError { - return malformedAuthError("incorrect service %q. This endpoint belongs to \"s3\".", s) - }, - MalformedComponent: func(s string) APIError { - return malformedAuthError("the authorization component %q is malformed.", s) - }, - MissingComponents: func() APIError { - return malformedAuthError("the authorization header requires three components: Credential, SignedHeaders, and Signature.") - }, - DateMismatch: func() APIError { - return malformedAuthError("The authorization header is malformed; Invalid credential date. Date is not the same as X-Amz-Date.") - }, +type malformedAuthErrors struct{} + +func (malformedAuthErrors) InvalidDateFormat(s string) APIError { + return malformedAuthError( + "incorrect date format %q. This date in the credential must be in the format \"yyyyMMdd\".", + s, + ) } + +func (malformedAuthErrors) MalformedCredential() APIError { + return malformedAuthError( + "the Credential is mal-formed; expecting \"/YYYYMMDD/REGION/SERVICE/aws4_request\".", + ) +} + +func (malformedAuthErrors) MissingCredential() APIError { + return malformedAuthError("missing Credential.") +} + +func (malformedAuthErrors) MissingSignature() APIError { + return malformedAuthError("missing Signature.") +} + +func (malformedAuthErrors) MissingSignedHeaders() APIError { + return malformedAuthError("missing SignedHeaders.") +} + +func (malformedAuthErrors) IncorrectTerminal(s string) APIError { + return malformedAuthError("incorrect terminal %q. This endpoint uses \"aws4_request\".", s) +} + +func (malformedAuthErrors) IncorrectRegion(expected, actual string) APIError { + return malformedAuthError("the region %q is wrong; expecting %q", actual, expected) +} + +func (malformedAuthErrors) IncorrectService(s string) APIError { + return malformedAuthError("incorrect service %q. This endpoint belongs to \"s3\".", s) +} + +func (malformedAuthErrors) MalformedComponent(s string) APIError { + return malformedAuthError("the authorization component %q is malformed.", s) +} + +func (malformedAuthErrors) MissingComponents() APIError { + return malformedAuthError( + "the authorization header requires three components: Credential, SignedHeaders, and Signature.", + ) +} + +func (malformedAuthErrors) DateMismatch() APIError { + return malformedAuthError( + "The authorization header is malformed; Invalid credential date. Date is not the same as X-Amz-Date.", + ) +} + +var MalformedAuth malformedAuthErrors diff --git a/s3response/s3response.go b/s3response/s3response.go index 63e44e43..9525d5fe 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -266,8 +266,18 @@ type Tagging struct { TagSet TagSet `xml:"TagSet"` } -type TaggingInput struct { - TagSet TagSet `xml:"TagSet"` +// UnmarshalXML accepts Tagging documents both with and without the S3 XML +// namespace, while xml.Marshal continues to emit the namespace via XMLName. +func (t *Tagging) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type plain struct { + TagSet TagSet `xml:"TagSet"` + } + var p plain + if err := d.DecodeElement(&p, &start); err != nil { + return err + } + t.TagSet = p.TagSet + return nil } type DeleteObjects struct { @@ -733,3 +743,11 @@ type CreateBucketConfiguration struct { LocationConstraint *string TagSet []types.Tag `xml:"Tags>Tag"` } + +type PostResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PostResponse"` + Location string + Bucket string + Key string + ETag string +} diff --git a/tests/integration/PutBucketTagging.go b/tests/integration/PutBucketTagging.go index f5e337d6..a9f60993 100644 --- a/tests/integration/PutBucketTagging.go +++ b/tests/integration/PutBucketTagging.go @@ -26,6 +26,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" ) func PutBucketTagging_non_existing_bucket(s *S3Conf) error { @@ -196,11 +197,13 @@ func PutBucketTagging_success(s *S3Conf) error { func PutBucketTagging_success_status(s *S3Conf) error { testName := "PutBucketTagging_success_status" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - tagging := types.Tagging{ - TagSet: []types.Tag{ - { - Key: getPtr("key"), - Value: getPtr("val"), + tagging := s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + { + Key: "key", + Value: "value", + }, }, }, } diff --git a/tests/integration/sigv4_auth.go b/tests/integration/sigv4_auth.go index a3530d69..90fb04d3 100644 --- a/tests/integration/sigv4_auth.go +++ b/tests/integration/sigv4_auth.go @@ -215,7 +215,7 @@ func Authentication_credentials_invalid_terminal(s *S3Conf) error { return err } - return checkHTTPResponseApiErr(resp, s3err.MalformedAuth.InvalidTerminal("aws_request")) + return checkHTTPResponseApiErr(resp, s3err.MalformedAuth.IncorrectTerminal("aws_request")) }) }