diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 102e94c..679d95b 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -27,7 +27,6 @@ import ( "math" "os" "path/filepath" - "slices" "sort" "strconv" "strings" @@ -1207,13 +1206,8 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete uncommittedBlocks[int32(ptNumber)] = el } - slices.SortFunc(blockList.UncommittedBlocks, func(a *blockblob.Block, b *blockblob.Block) int { - ptNumber, _ := decodeBlockId(*a.Name) - nextPtNumber, _ := decodeBlockId(*b.Name) - return ptNumber - nextPtNumber - }) - // The initialie values is the lower limit of partNumber: 0 + var totalSize int64 var partNumber int32 last := len(blockList.UncommittedBlocks) - 1 for i, part := range input.MultipartUpload.Parts { @@ -1241,9 +1235,14 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete if i < last && *block.Size < backend.MinPartSize { return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall) } + totalSize += *block.Size blockIds = append(blockIds, *block.Name) } + if input.MpuObjectSize != nil && totalSize != *input.MpuObjectSize { + return nil, s3err.GetIncorrectMpObjectSizeErr(totalSize, *input.MpuObjectSize) + } + opts := &blockblob.CommitBlockListOptions{ Metadata: props.Metadata, Tags: parseAzTags(tags.BlobTagSet), diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 5b3ac88..ef78065 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1478,6 +1478,10 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } } + if input.MpuObjectSize != nil && totalsize != *input.MpuObjectSize { + return nil, s3err.GetIncorrectMpObjectSizeErr(totalsize, *input.MpuObjectSize) + } + var hashRdr *utils.HashReader var compositeChecksumRdr *utils.CompositeChecksumReader switch checksums.Type { diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 9a496fb..069fb9e 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -3439,6 +3439,36 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { }) } + var mpuObjectSize *int64 + mpuObjSizeHdr := ctx.Get("X-Amz-Mp-Object-Size") + if mpuObjSizeHdr != "" { + val, err := strconv.ParseInt(mpuObjSizeHdr, 10, 64) + //TODO: Not sure if invalid request should be returned + if err != nil { + return SendXMLResponse(ctx, nil, + s3err.GetAPIError(s3err.ErrInvalidRequest), + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }) + } + + if val < 0 { + return SendXMLResponse(ctx, nil, + s3err.GetInvalidMpObjectSizeErr(val), + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }) + } + + mpuObjectSize = &val + } + err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, @@ -3497,6 +3527,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { MultipartUpload: &types.CompletedMultipartUpload{ Parts: data.Parts, }, + MpuObjectSize: mpuObjectSize, ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), diff --git a/s3err/s3err.go b/s3err/s3err.go index e0e9621..b5433b5 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -763,3 +763,19 @@ func GetChecksumTypeMismatchOnMpErr(t types.ChecksumType) APIError { HTTPStatusCode: http.StatusBadRequest, } } + +func GetIncorrectMpObjectSizeErr(expected, actual int64) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The provided 'x-amz-mp-object-size' header value %v does not match what was computed: %v", expected, actual), + HTTPStatusCode: http.StatusBadRequest, + } +} + +func GetInvalidMpObjectSizeErr(val int64) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("Value for x-amz-mp-object-size header is less than zero: '%v'", val), + HTTPStatusCode: http.StatusBadRequest, + } +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index f94dd40..f2b7937 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -394,6 +394,7 @@ func TestCompleteMultipartUpload(s *S3Conf) { CompleteMultipartUpload_small_upload_size(s) CompleteMultipartUpload_empty_parts(s) CompleteMultipartUpload_incorrect_parts_order(s) + CompleteMultipartUpload_mpu_object_size(s) //TODO: remove the condition after implementing checksums in azure if !s.azureTests { CompleteMultipartUpload_invalid_checksum_type(s) @@ -976,6 +977,7 @@ func GetIntTests() IntTests { "CompleteMultipartUpload_small_upload_size": CompleteMultipartUpload_small_upload_size, "CompleteMultipartUpload_empty_parts": CompleteMultipartUpload_empty_parts, "CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order, + "CompleteMultipartUpload_mpu_object_size": CompleteMultipartUpload_mpu_object_size, "CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type, "CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part, "CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index da3bde4..74bf738 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -9521,6 +9521,89 @@ func CompleteMultipartUpload_incorrect_parts_order(s *S3Conf) error { }) } +func CompleteMultipartUpload_mpu_object_size(s *S3Conf) error { + testName := "CompleteMultipartUpload_mpu_object_size" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + mpuSize := int64(23 * 1024 * 1024) // 23 mib + parts, _, err := uploadParts(s3client, mpuSize, 4, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + compParts := []types.CompletedPart{} + for _, el := range parts { + compParts = append(compParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + invMpuSize := int64(-1) // invalid MpuObjectSize + // Initially provide invalid MpuObjectSize: -3 + input := &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + MpuObjectSize: &invMpuSize, + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, input) + cancel() + if err := checkApiErr(err, s3err.GetInvalidMpObjectSizeErr(invMpuSize)); err != nil { + return err + } + + incorMpuSize := int64(213123) // incorrect object size + input.MpuObjectSize = &incorMpuSize + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, input) + cancel() + if err := checkApiErr(err, s3err.GetIncorrectMpObjectSizeErr(mpuSize, incorMpuSize)); err != nil { + return err + } + + // Correct value for MpuObjectSize + input.MpuObjectSize = &mpuSize + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, input) + cancel() + if err != nil { + return err + } + + // Make sure the object has been uploaded with proper size + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if res.ContentLength == nil { + return fmt.Errorf("expected non nil Content-Length") + } + if *res.ContentLength != mpuSize { + return fmt.Errorf("expected the uploaded object size to be %v, instead got %v", mpuSize, *res.ContentLength) + } + + return nil + }) +} + func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error { testName := "CompleteMultipartUpload_invalid_part_number" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {