From 756d155a62e9d41a96b09aa0389ae0ad6838ca6a Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Sat, 24 Aug 2024 14:31:36 -0700 Subject: [PATCH] fix: put file corruption with chunked transfer When on linux with O_TMPFILE support, we issue and fallocate for the expected object size ax an optimization for the underlying filesystem to allocate the full file all ate once. With the chunked transfer encoding, the final object size is recoded in the X-Amz-Decoded-Content-Length instead of the standard ContentLength which includes the chunk encoding in the payload. We were incorrectly using the content length to fallocate the file which would cause the filesystem to pad out any unwritten length to this size with 0s. The fix here is to make sure we pass the X-Amz-Decoded-Content-Length as the object size to the backend for all PUTs. Fixes #753 --- s3api/controllers/base.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 8804c3a..78ff020 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -1544,11 +1544,19 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { granWrite := ctx.Get("X-Amz-Grant-Write") grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") - // Other headers + // Content Length contentLengthStr := ctx.Get("Content-Length") if contentLengthStr == "" { contentLengthStr = "0" } + // Use decoded content length if available because the + // middleware will decode the chunked transfer encoding + decodedLength := ctx.Get("X-Amz-Decoded-Content-Length") + if decodedLength != "" { + contentLengthStr = decodedLength + } + + // Other headers bucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner") storageClass := ctx.Get("X-Amz-Storage-Class")