mirror of
https://github.com/versity/versitygw.git
synced 2026-10-22 14:22:40 +00:00
Ben McClelland
c354e1a2d3
Add request plumbing for server-side encryption with customer-provided keys (SSE-C) and stop silently ignoring it on backends that cannot honor it. Previously SSE-C headers were dropped before reaching the backend, so posix stored the data unencrypted and returned success, and later reads without the key succeeded. Clients believed their data was encrypted when it was not. Partial or malformed SSE-C header sets were also accepted instead of failing with 400 as S3 does. Rejecting SSE-C explicitly is the safe behavior until posix implements it, and azure cannot faithfully map SSE-C onto customer-provided keys. This addresses part of #2193: invalid/partial SSE-C headers now fail with 400. Requiring SSE-C headers on reads of SSE-C objects depends on posix SSE-C support and remains open. The changes also reject unsupported SSE-S3, SSE-KMS, and DSSE-KMS requests instead of silently accepting them. And enforcea TLS for SSE-C by default, with --disable-sse-c-tls-enforcement as the opt-out in case the deployment handles TLS in front of the gateway.
631 lines
16 KiB
Go
631 lines
16 KiB
Go
// Copyright 2023 Versity Software
|
|
// This file is licensed under the Apache License, Version 2.0
|
|
// (the "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/xml"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/versity/versitygw/s3api/utils"
|
|
"github.com/versity/versitygw/s3err"
|
|
"github.com/versity/versitygw/s3event"
|
|
"github.com/versity/versitygw/s3response"
|
|
)
|
|
|
|
func TestS3ApiController_RestoreObject(t *testing.T) {
|
|
validRestoreBody, err := xml.Marshal(types.RestoreRequest{
|
|
Description: utils.GetStringPtr("description"),
|
|
Type: types.RestoreRequestTypeSelect,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
input testInput
|
|
output testOutput
|
|
}{
|
|
{
|
|
name: "verify access fails",
|
|
input: testInput{
|
|
locals: accessDeniedLocals,
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrAccessDenied),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid request body",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: []byte("invalid_body"),
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrMalformedXML),
|
|
},
|
|
},
|
|
{
|
|
name: "backend returns error",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
|
body: validRestoreBody,
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
EventName: s3event.EventObjectRestoreCompleted,
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
|
},
|
|
},
|
|
{
|
|
name: "successful response",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validRestoreBody,
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
EventName: s3event.EventObjectRestoreCompleted,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
be := &BackendMock{
|
|
RestoreObjectFunc: func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error {
|
|
return tt.input.beErr
|
|
},
|
|
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
|
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
|
},
|
|
}
|
|
|
|
ctrl := S3ApiController{
|
|
be: be,
|
|
}
|
|
|
|
testController(
|
|
t,
|
|
ctrl.RestoreObject,
|
|
tt.output.response,
|
|
tt.output.err,
|
|
ctxInputs{
|
|
locals: tt.input.locals,
|
|
body: tt.input.body,
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestS3ApiController_SelectObjectContent(t *testing.T) {
|
|
validSelectBody, err := xml.Marshal(s3response.SelectObjectContentPayload{
|
|
Expression: utils.GetStringPtr("expression"),
|
|
ExpressionType: types.ExpressionTypeSql,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
input testInput
|
|
output testOutput
|
|
}{
|
|
{
|
|
name: "verify access fails",
|
|
input: testInput{
|
|
locals: accessDeniedLocals,
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrAccessDenied),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid request body",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: []byte("invalid_body"),
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrMalformedXML),
|
|
},
|
|
},
|
|
{
|
|
name: "successful response",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validSelectBody,
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
be := &BackendMock{
|
|
SelectObjectContentFunc: func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
|
return func(w *bufio.Writer) {}
|
|
},
|
|
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
|
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
|
},
|
|
}
|
|
|
|
ctrl := S3ApiController{
|
|
be: be,
|
|
}
|
|
|
|
testController(
|
|
t,
|
|
ctrl.SelectObjectContent,
|
|
tt.output.response,
|
|
tt.output.err,
|
|
ctxInputs{
|
|
locals: tt.input.locals,
|
|
body: tt.input.body,
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestS3ApiController_CreateMultipartUpload(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input testInput
|
|
output testOutput
|
|
}{
|
|
{
|
|
name: "verify access fails",
|
|
input: testInput{
|
|
locals: accessDeniedLocals,
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrAccessDenied),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid metadata header",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
headers: map[string]string{
|
|
"x-amz-meta-key": strings.Repeat("a", 2048),
|
|
},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetMetadataTooLargeErr(2051, 2048),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid object lock headers",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
headers: map[string]string{
|
|
"X-Amz-Object-Lock-Mode": string(types.ObjectLockModeGovernance),
|
|
},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingObjectLockRetainDate, ""),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid checksum headers",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
headers: map[string]string{
|
|
"X-Amz-Checksum-Algorithm": "invalid_checksum_algo",
|
|
},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm),
|
|
},
|
|
},
|
|
{
|
|
name: "backend returns error",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
|
beRes: s3response.InitiateMultipartUploadResult{},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
Data: s3response.InitiateMultipartUploadResult{},
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
|
},
|
|
},
|
|
{
|
|
name: "successful response",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
beRes: s3response.InitiateMultipartUploadResult{},
|
|
headers: map[string]string{
|
|
"x-amz-checksum-algorithm": string(types.ChecksumAlgorithmCrc32),
|
|
"x-amz-checksum-type": string(types.ChecksumTypeComposite),
|
|
"X-Amz-Storage-Class": string(types.StorageClassGlacier),
|
|
},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
Data: s3response.InitiateMultipartUploadResult{},
|
|
Headers: map[string]*string{
|
|
"x-amz-checksum-algorithm": utils.ConvertToStringPtr(types.ChecksumAlgorithmCrc32),
|
|
"x-amz-checksum-type": utils.ConvertToStringPtr(types.ChecksumTypeComposite),
|
|
"x-amz-server-side-encryption-customer-algorithm": nil,
|
|
"x-amz-server-side-encryption-customer-key-MD5": nil,
|
|
},
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
be := &BackendMock{
|
|
CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
|
|
if tt.name == "successful response" && createMultipartUploadInput.StorageClass != types.StorageClassGlacier {
|
|
t.Fatalf("expected storage class %q, got %q", types.StorageClassGlacier, createMultipartUploadInput.StorageClass)
|
|
}
|
|
return tt.input.beRes.(s3response.InitiateMultipartUploadResult), tt.input.beErr
|
|
},
|
|
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
|
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
|
},
|
|
}
|
|
|
|
ctrl := S3ApiController{
|
|
be: be,
|
|
}
|
|
|
|
testController(
|
|
t,
|
|
ctrl.CreateMultipartUpload,
|
|
tt.output.response,
|
|
tt.output.err,
|
|
ctxInputs{
|
|
locals: tt.input.locals,
|
|
body: tt.input.body,
|
|
headers: tt.input.headers,
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestS3ApiController_CompleteMultipartUpload(t *testing.T) {
|
|
emptyMpPartsBody, err := xml.Marshal(s3response.CompleteMultipartUploadRequestBody{
|
|
Parts: []types.CompletedPart{},
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
validMpBody := []byte(`<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Part><PartNumber>1</PartNumber><ETag>ETag</ETag></Part></CompleteMultipartUpload>`)
|
|
s3cmdMpBody := []byte("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>ETag</ETag></Part></CompleteMultipartUpload>")
|
|
|
|
versionId, ETag := "versionId", "mock-ETag"
|
|
|
|
tests := []struct {
|
|
name string
|
|
input testInput
|
|
output testOutput
|
|
}{
|
|
{
|
|
name: "verify access fails",
|
|
input: testInput{
|
|
locals: accessDeniedLocals,
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrAccessDenied),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid request body",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: []byte("invalid_body"),
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrMalformedXML),
|
|
},
|
|
},
|
|
{
|
|
name: "request body empty mp parts",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: emptyMpPartsBody,
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrMalformedXML),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid mp parts header string",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validMpBody,
|
|
headers: map[string]string{
|
|
"X-Amz-Mp-Object-Size": "invalid_mp_object_size",
|
|
},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetInvalidMpObjectSizeErr("invalid_mp_object_size"),
|
|
},
|
|
},
|
|
{
|
|
name: "negative mp parts header value",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validMpBody,
|
|
headers: map[string]string{
|
|
"X-Amz-Mp-Object-Size": "-4",
|
|
},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetNegatvieMpObjectSizeErr(-4),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid checksum headers",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validMpBody,
|
|
headers: map[string]string{
|
|
"X-Amz-Checksum-Crc32": "invalid_checksum",
|
|
},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"),
|
|
},
|
|
},
|
|
{
|
|
name: "invalid checksum type",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validMpBody,
|
|
headers: map[string]string{
|
|
"X-Amz-Checksum-Type": "invalid_checksum_type",
|
|
},
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type"),
|
|
},
|
|
},
|
|
{
|
|
name: "object is locked",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validMpBody,
|
|
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLocked),
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrObjectLocked),
|
|
},
|
|
},
|
|
{
|
|
name: "backend returns error",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validMpBody,
|
|
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
|
beRes: s3response.CompleteMultipartUploadResult{},
|
|
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
Data: s3response.CompleteMultipartUploadResult{},
|
|
Headers: map[string]*string{
|
|
"x-amz-version-id": &versionId,
|
|
},
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
EventName: s3event.EventCompleteMultipartUpload,
|
|
VersionId: &versionId,
|
|
ObjectETag: nil,
|
|
},
|
|
},
|
|
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
|
},
|
|
},
|
|
{
|
|
name: "successful response",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: validMpBody,
|
|
beRes: s3response.CompleteMultipartUploadResult{
|
|
ETag: &ETag,
|
|
},
|
|
headers: map[string]string{
|
|
"X-Amz-Mp-Object-Size": "3",
|
|
},
|
|
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
Data: s3response.CompleteMultipartUploadResult{
|
|
ETag: &ETag,
|
|
Location: utils.GetStringPtr("http://example.com/bucket/object"),
|
|
},
|
|
Headers: map[string]*string{
|
|
"x-amz-version-id": &versionId,
|
|
},
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
EventName: s3event.EventCompleteMultipartUpload,
|
|
VersionId: &versionId,
|
|
ObjectETag: &ETag,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "successful response with s3cmd request body",
|
|
input: testInput{
|
|
locals: defaultLocals,
|
|
body: s3cmdMpBody,
|
|
beRes: s3response.CompleteMultipartUploadResult{ETag: &ETag},
|
|
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
|
},
|
|
output: testOutput{
|
|
response: &Response{
|
|
Data: s3response.CompleteMultipartUploadResult{
|
|
ETag: &ETag,
|
|
Location: utils.GetStringPtr("http://example.com/bucket/object"),
|
|
},
|
|
Headers: map[string]*string{
|
|
"x-amz-version-id": &versionId,
|
|
},
|
|
MetaOpts: &MetaOptions{
|
|
BucketOwner: "root",
|
|
EventName: s3event.EventCompleteMultipartUpload,
|
|
VersionId: &versionId,
|
|
ObjectETag: &ETag,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
be := &BackendMock{
|
|
CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
|
|
return tt.input.beRes.(s3response.CompleteMultipartUploadResult), versionId, tt.input.beErr
|
|
},
|
|
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
|
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
|
},
|
|
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
|
return nil, tt.input.extraMockErr
|
|
},
|
|
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
|
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
|
},
|
|
}
|
|
|
|
ctrl := S3ApiController{
|
|
be: be,
|
|
}
|
|
|
|
testController(
|
|
t,
|
|
ctrl.CompleteMultipartUpload,
|
|
tt.output.response,
|
|
tt.output.err,
|
|
ctxInputs{
|
|
locals: tt.input.locals,
|
|
body: tt.input.body,
|
|
headers: tt.input.headers,
|
|
})
|
|
})
|
|
}
|
|
}
|