Files
versitygw/s3api/controllers/object-post_test.go
T
niksis02 008b8a49ba fix: return NoSuchUpload for malformed upload IDs ahead of access and request checks
Fixes #2533

`UploadPartCopy` and `CopyObject` returned `500 InternalError` when a non-root user copied from a source bucket with no stored ACL (such as a posix bucket directory created outside the gateway): `VerifyObjectCopyAccess` ran `json.Unmarshal` on the empty ACL bytes posix returns for such a bucket. It now parses them with `ParseACL`, like every other ACL reader.

The source checks also ran before the backend looked at the upload ID, so a malformed `uploadId` got `AccessDenied`, `NoSuchBucket` or `InvalidArgument` instead of `NoSuchUpload`. S3 rejects an upload ID that is malformed, or bound to another key, right after the bucket existence and expected-bucket-owner checks, before authorization and nearly all request validation. That applies to every multipart operation that takes an upload ID.

`Backend` gains `ValidateUploadID`, a format-only check that doesn't look up the upload. `BackendUnsupported` accepts every ID, so s3proxy and azure are unchanged, and posix's existing canonical UUID check becomes the method. `UploadPart`, `UploadPartCopy` and `AbortMultipartUpload` call it first. `ListParts` calls it after rejecting a `part-number-marker` or `max-parts` that isn't an integer (new `ValidateLimiterInteger`); negative values are still rejected after it. `CompleteMultipartUpload` calls it after rejecting unsupported `If-Match`/`If-None-Match` combinations, which are split out of `EvaluateObjectPutPreconditions` as `ValidateObjectPutPreconditionHeaders` and now checked before access verification and body parsing, as on S3.
2026-10-21 19:55:15 +04:00

677 lines
17 KiB
Go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package controllers
import (
"bufio"
"context"
"encoding/xml"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3response"
)
func TestS3ApiController_RestoreObject(t *testing.T) {
validRestoreBody, err := xml.Marshal(types.RestoreRequest{
Description: utils.GetStringPtr("description"),
Type: types.RestoreRequestTypeSelect,
})
assert.NoError(t, err)
tests := []struct {
name string
input testInput
output testOutput
}{
{
name: "verify access fails",
input: testInput{
locals: accessDeniedLocals,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrAccessDenied),
},
},
{
name: "invalid request body",
input: testInput{
locals: defaultLocals,
body: []byte("invalid_body"),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMalformedXML),
},
},
{
name: "backend returns error",
input: testInput{
locals: defaultLocals,
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
body: validRestoreBody,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventObjectRestoreCompleted,
},
},
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
},
},
{
name: "successful response",
input: testInput{
locals: defaultLocals,
body: validRestoreBody,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventObjectRestoreCompleted,
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
be := &BackendMock{
RestoreObjectFunc: func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error {
return tt.input.beErr
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
},
}
ctrl := S3ApiController{
be: be,
}
testController(
t,
ctrl.RestoreObject,
tt.output.response,
tt.output.err,
ctxInputs{
locals: tt.input.locals,
body: tt.input.body,
})
})
}
}
func TestS3ApiController_SelectObjectContent(t *testing.T) {
validSelectBody, err := xml.Marshal(s3response.SelectObjectContentPayload{
Expression: utils.GetStringPtr("expression"),
ExpressionType: types.ExpressionTypeSql,
})
assert.NoError(t, err)
tests := []struct {
name string
input testInput
output testOutput
}{
{
name: "verify access fails",
input: testInput{
locals: accessDeniedLocals,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrAccessDenied),
},
},
{
name: "invalid request body",
input: testInput{
locals: defaultLocals,
body: []byte("invalid_body"),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMalformedXML),
},
},
{
name: "successful response",
input: testInput{
locals: defaultLocals,
body: validSelectBody,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
be := &BackendMock{
SelectObjectContentFunc: func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
return func(w *bufio.Writer) {}
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
},
}
ctrl := S3ApiController{
be: be,
}
testController(
t,
ctrl.SelectObjectContent,
tt.output.response,
tt.output.err,
ctxInputs{
locals: tt.input.locals,
body: tt.input.body,
})
})
}
}
func TestS3ApiController_CreateMultipartUpload(t *testing.T) {
tests := []struct {
name string
input testInput
output testOutput
}{
{
name: "verify access fails",
input: testInput{
locals: accessDeniedLocals,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrAccessDenied),
},
},
{
name: "invalid metadata header",
input: testInput{
locals: defaultLocals,
headers: map[string]string{
"x-amz-meta-key": strings.Repeat("a", 2048),
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetMetadataTooLargeErr(2051, 2048),
},
},
{
name: "invalid object lock headers",
input: testInput{
locals: defaultLocals,
headers: map[string]string{
"X-Amz-Object-Lock-Mode": string(types.ObjectLockModeGovernance),
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingObjectLockRetainDate, ""),
},
},
{
name: "invalid checksum headers",
input: testInput{
locals: defaultLocals,
headers: map[string]string{
"X-Amz-Checksum-Algorithm": "invalid_checksum_algo",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm),
},
},
{
name: "backend returns error",
input: testInput{
locals: defaultLocals,
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
beRes: s3response.InitiateMultipartUploadResult{},
},
output: testOutput{
response: &Response{
Data: s3response.InitiateMultipartUploadResult{},
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
},
},
{
name: "successful response",
input: testInput{
locals: defaultLocals,
beRes: s3response.InitiateMultipartUploadResult{},
headers: map[string]string{
"x-amz-checksum-algorithm": string(types.ChecksumAlgorithmCrc32),
"x-amz-checksum-type": string(types.ChecksumTypeComposite),
"X-Amz-Storage-Class": string(types.StorageClassGlacier),
},
},
output: testOutput{
response: &Response{
Data: s3response.InitiateMultipartUploadResult{},
Headers: map[string]*string{
"x-amz-checksum-algorithm": utils.ConvertToStringPtr(types.ChecksumAlgorithmCrc32),
"x-amz-checksum-type": utils.ConvertToStringPtr(types.ChecksumTypeComposite),
"x-amz-server-side-encryption-customer-algorithm": nil,
"x-amz-server-side-encryption-customer-key-MD5": nil,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
be := &BackendMock{
CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
if tt.name == "successful response" && createMultipartUploadInput.StorageClass != types.StorageClassGlacier {
t.Fatalf("expected storage class %q, got %q", types.StorageClassGlacier, createMultipartUploadInput.StorageClass)
}
return tt.input.beRes.(s3response.InitiateMultipartUploadResult), tt.input.beErr
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
},
}
ctrl := S3ApiController{
be: be,
}
testController(
t,
ctrl.CreateMultipartUpload,
tt.output.response,
tt.output.err,
ctxInputs{
locals: tt.input.locals,
body: tt.input.body,
headers: tt.input.headers,
})
})
}
}
func TestS3ApiController_CompleteMultipartUpload(t *testing.T) {
emptyMpPartsBody, err := xml.Marshal(s3response.CompleteMultipartUploadRequestBody{
Parts: []types.CompletedPart{},
})
assert.NoError(t, err)
validMpBody := []byte(`<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Part><PartNumber>1</PartNumber><ETag>ETag</ETag></Part></CompleteMultipartUpload>`)
s3cmdMpBody := []byte("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>ETag</ETag></Part></CompleteMultipartUpload>")
versionId, ETag := "versionId", "mock-ETag"
tests := []struct {
name string
input testInput
output testOutput
}{
{
name: "unsupported conditional header",
input: testInput{
locals: accessDeniedLocals,
body: []byte("invalid_body"),
headers: map[string]string{
"If-None-Match": "ETag",
},
queries: map[string]string{
"uploadId": "bad",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetNotImplementedErr("If-None-Match", s3err.NmpAdditionalMessageIfNoneMatch),
},
},
{
name: "malformed upload id",
input: testInput{
locals: accessDeniedLocals,
body: []byte("invalid_body"),
queries: map[string]string{
"uploadId": "bad",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetNoSuchUploadErr("bad"),
},
},
{
name: "verify access fails",
input: testInput{
locals: accessDeniedLocals,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrAccessDenied),
},
},
{
name: "invalid request body",
input: testInput{
locals: defaultLocals,
body: []byte("invalid_body"),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMalformedXML),
},
},
{
name: "request body empty mp parts",
input: testInput{
locals: defaultLocals,
body: emptyMpPartsBody,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMalformedXML),
},
},
{
name: "invalid mp parts header string",
input: testInput{
locals: defaultLocals,
body: validMpBody,
headers: map[string]string{
"X-Amz-Mp-Object-Size": "invalid_mp_object_size",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetInvalidMpObjectSizeErr("invalid_mp_object_size"),
},
},
{
name: "negative mp parts header value",
input: testInput{
locals: defaultLocals,
body: validMpBody,
headers: map[string]string{
"X-Amz-Mp-Object-Size": "-4",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetNegatvieMpObjectSizeErr(-4),
},
},
{
name: "invalid checksum headers",
input: testInput{
locals: defaultLocals,
body: validMpBody,
headers: map[string]string{
"X-Amz-Checksum-Crc32": "invalid_checksum",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"),
},
},
{
name: "invalid checksum type",
input: testInput{
locals: defaultLocals,
body: validMpBody,
headers: map[string]string{
"X-Amz-Checksum-Type": "invalid_checksum_type",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type"),
},
},
{
name: "object is locked",
input: testInput{
locals: defaultLocals,
body: validMpBody,
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLocked),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrObjectLocked),
},
},
{
name: "backend returns error",
input: testInput{
locals: defaultLocals,
body: validMpBody,
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
beRes: s3response.CompleteMultipartUploadResult{},
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: s3response.CompleteMultipartUploadResult{},
Headers: map[string]*string{
"x-amz-version-id": &versionId,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventCompleteMultipartUpload,
VersionId: &versionId,
ObjectETag: nil,
},
},
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
},
},
{
name: "successful response",
input: testInput{
locals: defaultLocals,
body: validMpBody,
beRes: s3response.CompleteMultipartUploadResult{
ETag: &ETag,
},
headers: map[string]string{
"X-Amz-Mp-Object-Size": "3",
},
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: s3response.CompleteMultipartUploadResult{
ETag: &ETag,
Location: utils.GetStringPtr("http://example.com/bucket/object"),
},
Headers: map[string]*string{
"x-amz-version-id": &versionId,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventCompleteMultipartUpload,
VersionId: &versionId,
ObjectETag: &ETag,
},
},
},
},
{
name: "successful response with s3cmd request body",
input: testInput{
locals: defaultLocals,
body: s3cmdMpBody,
beRes: s3response.CompleteMultipartUploadResult{ETag: &ETag},
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: s3response.CompleteMultipartUploadResult{
ETag: &ETag,
Location: utils.GetStringPtr("http://example.com/bucket/object"),
},
Headers: map[string]*string{
"x-amz-version-id": &versionId,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventCompleteMultipartUpload,
VersionId: &versionId,
ObjectETag: &ETag,
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
be := &BackendMock{
ValidateUploadIDFunc: func(uploadID string) error {
if uploadID == "bad" {
return s3err.GetNoSuchUploadErr(uploadID)
}
return nil
},
CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
return tt.input.beRes.(s3response.CompleteMultipartUploadResult), versionId, tt.input.beErr
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, tt.input.extraMockErr
},
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
},
}
ctrl := S3ApiController{
be: be,
}
testController(
t,
ctrl.CompleteMultipartUpload,
tt.output.response,
tt.output.err,
ctxInputs{
locals: tt.input.locals,
body: tt.input.body,
headers: tt.input.headers,
queries: tt.input.queries,
})
})
}
}