diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index 407e8e16..421f3abd 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -147,7 +147,7 @@ func (c S3ApiController) CreateMultipartUpload(ctx fiber.Ctx) (*Response, error) contentDisposition := ctx.Get("Content-Disposition") contentLanguage := ctx.Get("Content-Language") cacheControl := ctx.Get("Cache-Control") - contentEncoding := ctx.Get("Content-Encoding") + contentEncoding := utils.StripAwsChunkedEncoding(ctx.Get("Content-Encoding")) tagging := ctx.Get("X-Amz-Tagging") expires := ctx.Get("Expires") websiteRedirectLocation := ctx.Get("X-Amz-Website-Redirect-Location") diff --git a/s3api/controllers/object-post_test.go b/s3api/controllers/object-post_test.go index dd5253d8..c6766ccb 100644 --- a/s3api/controllers/object-post_test.go +++ b/s3api/controllers/object-post_test.go @@ -328,6 +328,28 @@ func TestS3ApiController_CreateMultipartUpload(t *testing.T) { }, }, }, + { + name: "strips aws-chunked content encoding", + input: testInput{ + locals: defaultLocals, + beRes: s3response.InitiateMultipartUploadResult{}, + headers: map[string]string{ + "Content-Encoding": "aws-chunked,gzip", + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.InitiateMultipartUploadResult{}, + Headers: map[string]*string{ + "x-amz-checksum-algorithm": nil, + "x-amz-checksum-type": nil, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -336,6 +358,14 @@ func TestS3ApiController_CreateMultipartUpload(t *testing.T) { if tt.name == "successful response" && createMultipartUploadInput.StorageClass != types.StorageClassGlacier { t.Fatalf("expected storage class %q, got %q", types.StorageClassGlacier, createMultipartUploadInput.StorageClass) } + if tt.name == "strips aws-chunked content encoding" { + if createMultipartUploadInput.ContentEncoding == nil { + t.Fatal("expected content encoding to be set") + } + if *createMultipartUploadInput.ContentEncoding != "gzip" { + t.Fatalf("expected content encoding %q, got %q", "gzip", *createMultipartUploadInput.ContentEncoding) + } + } return tt.input.beRes.(s3response.InitiateMultipartUploadResult), tt.input.beErr }, GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index c07dd072..a1317921 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -516,7 +516,7 @@ func (c S3ApiController) CopyObject(ctx fiber.Ctx) (*Response, error) { metaDirective := types.MetadataDirective(ctx.Get("X-Amz-Metadata-Directive", string(types.MetadataDirectiveCopy))) taggingDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive", string(types.TaggingDirectiveCopy))) contentType := ctx.Get("Content-Type", defaultContentType) - contentEncoding := ctx.Get("Content-Encoding") + contentEncoding := utils.StripAwsChunkedEncoding(ctx.Get("Content-Encoding")) contentDisposition := ctx.Get("Content-Disposition") contentLanguage := ctx.Get("Content-Language") cacheControl := ctx.Get("Cache-Control") @@ -698,7 +698,7 @@ func (c S3ApiController) PutObject(ctx fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) contentType := ctx.Get("Content-Type", defaultContentType) - contentEncoding := ctx.Get("Content-Encoding") + contentEncoding := utils.StripAwsChunkedEncoding(ctx.Get("Content-Encoding")) contentDisposition := ctx.Get("Content-Disposition") contentLanguage := ctx.Get("Content-Language") cacheControl := ctx.Get("Cache-Control") diff --git a/s3api/controllers/object-put_test.go b/s3api/controllers/object-put_test.go index 14611a2d..0f72fa9d 100644 --- a/s3api/controllers/object-put_test.go +++ b/s3api/controllers/object-put_test.go @@ -1331,6 +1331,61 @@ func TestS3ApiController_PutObject(t *testing.T) { }) }) + t.Run("strips aws-chunked content encoding", func(t *testing.T) { + be := &BackendMock{ + PutObjectFunc: func(_ context.Context, input s3response.PutObjectInput) (s3response.PutObjectOutput, error) { + if input.ContentEncoding == nil { + t.Fatal("expected content encoding to be set") + } + if *input.ContentEncoding != "gzip" { + t.Fatalf("expected content encoding %q, got %q", "gzip", *input.ContentEncoding) + } + return s3response.PutObjectOutput{ETag: "etag", VersionID: "version-id"}, nil + }, + GetBucketPolicyFunc: func(_ context.Context, _ string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + GetObjectLockConfigurationFunc: func(_ context.Context, _ string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) + }, + GetBucketVersioningFunc: func(_ context.Context, _ string) (s3response.GetBucketVersioningOutput, error) { + return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented) + }, + } + + ctrl := S3ApiController{be: be} + testController(t, ctrl.PutObject, &Response{ + Headers: map[string]*string{ + "ETag": utils.GetStringPtr("etag"), + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, + "x-amz-checksum-type": nil, + "x-amz-version-id": utils.GetStringPtr("version-id"), + "x-amz-object-size": nil, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ObjectETag: utils.GetStringPtr("etag"), + ContentLength: 0, + ObjectSize: 0, + EventName: s3event.EventObjectCreatedPut, + }, + }, nil, ctxInputs{ + locals: defaultLocals, + headers: map[string]string{ + "Content-Encoding": "aws-chunked,gzip", + }, + }) + }) + tests := []struct { name string input testInput diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index f0b42bdc..3ad79622 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -1112,3 +1112,28 @@ func ValidateLocationConstraint(constraint *string, region string) error { return nil } + +// The coding announced when a body is framed in aws-chunked, as the SDKs do to +// carry a trailing checksum. +const awsChunkedEncoding = "aws-chunked" + +// StripAwsChunkedEncoding drops the aws-chunked token, which frames the request +// rather than the object, from a Content-Encoding value. +func StripAwsChunkedEncoding(contentEncoding string) string { + if contentEncoding == "" { + return "" + } + + codings := strings.Split(contentEncoding, ",") + kept := make([]string, 0, len(codings)) + for _, coding := range codings { + trimmed := strings.TrimSpace(coding) + if trimmed == "" || strings.EqualFold(trimmed, awsChunkedEncoding) { + continue + } + + kept = append(kept, trimmed) + } + + return strings.Join(kept, ",") +} diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 10390de7..02a17ddf 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -1515,3 +1515,30 @@ func TestValidateCopySource(t *testing.T) { }) } } + +func TestStripAwsChunkedEncoding(t *testing.T) { + tests := []struct { + name string + contentEncoding string + want string + }{ + {"empty", "", ""}, + {"only aws-chunked", "aws-chunked", ""}, + {"only aws-chunked, uppercase", "AWS-CHUNKED", ""}, + {"only aws-chunked, padded", " aws-chunked ", ""}, + {"no aws-chunked", "gzip", "gzip"}, + {"other codings kept in order", "deflate,gzip", "deflate,gzip"}, + {"aws-chunked first", "aws-chunked,gzip", "gzip"}, + {"aws-chunked last", "gzip,aws-chunked", "gzip"}, + {"aws-chunked in the middle", "deflate,aws-chunked,gzip", "deflate,gzip"}, + {"spaces around codings", "aws-chunked, gzip", "gzip"}, + {"repeated aws-chunked", "aws-chunked,aws-chunked", ""}, + {"empty coding dropped", "gzip,,aws-chunked", "gzip"}, + {"coding containing the token is kept", "aws-chunked-custom", "aws-chunked-custom"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, StripAwsChunkedEncoding(tt.contentEncoding)) + }) + } +}