From f849b7c823ed781c5210f01056e62157eb04018f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 19 Sep 2026 22:18:13 -0700 Subject: [PATCH] s3: validate per-part checksums in CompleteMultipartUpload Parse the Checksum* elements of each completed part and enforce what AWS does for uploads created with x-amz-checksum-algorithm: every part must carry a checksum in the complete request (InvalidRequest when missing, BadDigest when it differs from the stored part checksum), and an x-amz-checksum-type header must match the upload resolved checksum type (BadDigest). Add the issue-11401 reproduction as a regression test. --- test/s3/checksum/issue11401_repro_test.go | 128 ++++++++++++++++++ weed/s3api/filer_multipart.go | 25 +++- weed/s3api/s3api_object_handlers_multipart.go | 1 + weed/s3api/s3api_object_handlers_put.go | 16 +++ 4 files changed, 168 insertions(+), 2 deletions(-) diff --git a/test/s3/checksum/issue11401_repro_test.go b/test/s3/checksum/issue11401_repro_test.go index f6ee2e49b..3aee839c3 100644 --- a/test/s3/checksum/issue11401_repro_test.go +++ b/test/s3/checksum/issue11401_repro_test.go @@ -3,6 +3,9 @@ package checksum_test import ( "bytes" "context" + "crypto/sha256" + "encoding/base64" + "errors" "testing" "github.com/aws/aws-sdk-go-v2/aws" @@ -10,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/stretchr/testify/require" ) @@ -83,3 +87,127 @@ func TestMultipartPartInheritsChecksumAlgorithm(t *testing.T) { require.Equal(t, types.ChecksumTypeComposite, done.ChecksumType) require.NotEmpty(t, aws.ToString(done.ChecksumSHA256)) } + +// Mimics the .NET repro in https://github.com/seaweedfs/seaweedfs/issues/11401: +// UploadPart carries an explicit client-computed x-amz-checksum-sha256 value, +// CompleteMultipartUpload echoes it back per part, and HeadObject returns the +// object checksum when x-amz-checksum-mode: ENABLED is sent (same as AWS). +func TestIssue11401(t *testing.T) { + client := getS3Client(t) + + bucket := uniqueBucket() + createBucket(t, client, bucket) + defer cleanupBucket(t, client, bucket) + + body := bytes.Repeat([]byte("x"), 1024) + checksum := base64.StdEncoding.EncodeToString(func() []byte { s := sha256.Sum256(body); return s[:] }()) + key := "issue-11401" + + create, err := client.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + ContentType: aws.String("application/octet-stream"), + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }) + require.NoError(t, err) + + part, err := client.UploadPart(context.Background(), &s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: create.UploadId, + PartNumber: aws.Int32(1), + Body: bytes.NewReader(body), + ChecksumSHA256: aws.String(checksum), + }) + require.NoError(t, err) + require.Equal(t, checksum, aws.ToString(part.ChecksumSHA256)) + + done, err := client.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: create.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{{ + ETag: part.ETag, + PartNumber: aws.Int32(1), + ChecksumSHA256: part.ChecksumSHA256, + }}}, + }) + require.NoError(t, err) + require.Equal(t, types.ChecksumTypeComposite, done.ChecksumType) + require.NotEmpty(t, aws.ToString(done.ChecksumSHA256)) + + head, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + ChecksumMode: types.ChecksumModeEnabled, + }) + require.NoError(t, err) + require.Equal(t, aws.ToString(done.ChecksumSHA256), aws.ToString(head.ChecksumSHA256)) + + headNoMode, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + require.NoError(t, err) + require.Empty(t, aws.ToString(headNoMode.ChecksumSHA256)) +} + +// CompleteMultipartUpload validates the per-part checksums it is given, like +// AWS: missing checksums fail with InvalidRequest and wrong ones with +// BadDigest. +func TestCompleteMultipartUploadValidatesPartChecksums(t *testing.T) { + client := newWhenRequiredChecksumClient(t) + + bucket := uniqueBucket() + createBucket(t, client, bucket) + defer cleanupBucket(t, client, bucket) + + body := bytes.Repeat([]byte("z"), 1024) + key := "validate-parts" + + create, err := client.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }) + require.NoError(t, err) + + part, err := client.UploadPart(context.Background(), &s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: create.UploadId, + PartNumber: aws.Int32(1), + Body: bytes.NewReader(body), + }) + require.NoError(t, err) + require.NotEmpty(t, aws.ToString(part.ChecksumSHA256)) + + complete := func(checksum *string) error { + _, err := client.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: create.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{{ + ETag: part.ETag, + PartNumber: aws.Int32(1), + ChecksumSHA256: checksum, + }}}, + }) + return err + } + + var apiErr smithy.APIError + + err = complete(nil) + require.Error(t, err) + require.True(t, errors.As(err, &apiErr)) + require.Equal(t, "InvalidRequest", apiErr.ErrorCode()) + + wrong := base64.StdEncoding.EncodeToString(func() []byte { s := sha256.Sum256([]byte("other")); return s[:] }()) + err = complete(aws.String(wrong)) + require.Error(t, err) + require.True(t, errors.As(err, &apiErr)) + require.Equal(t, "BadDigest", apiErr.ErrorCode()) + + require.NoError(t, complete(part.ChecksumSHA256)) +} diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 67272b3cd..f9fdc763f 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -383,7 +383,7 @@ func applyMultipartSSES3HeadersFromUploadEntry(dst *filer_pb.Entry, sses3Info *m } } -func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *s3.CompleteMultipartUploadInput, uploadDirectory, entryName, dirName string, completedPartNumbers []int, completedPartMap map[int][]string, maxPartNo int) (*multipartCompletionState, *CompleteMultipartUploadResult, s3err.ErrorCode) { +func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload, uploadDirectory, entryName, dirName string, completedPartNumbers []int, completedPartMap map[int][]string, maxPartNo int) (*multipartCompletionState, *CompleteMultipartUploadResult, s3err.ErrorCode) { if entry, err := s3a.resolveObjectEntry(*input.Bucket, *input.Key, ""); err == nil && entry != nil && entry.Extended != nil { if uploadId, ok := entry.Extended[s3_constants.SeaweedFSUploadId]; ok && *input.UploadId == string(uploadId) { cleanupEntries, _, cleanupErr := s3a.list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1) @@ -581,8 +581,29 @@ func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input * glog.Errorf("completeMultipartUpload: %v", typeErr) return nil, nil, s3err.ErrInvalidRequest } + if completeType := r.Header.Get(s3_constants.AmzChecksumType); completeType != "" && !strings.EqualFold(completeType, resolvedType) { + return nil, nil, s3err.ErrBadDigest + } checksumType = resolvedType + provided := make(map[int]string, len(parts.Parts)) + for _, part := range parts.Parts { + provided[part.PartNumber] = part.GetChecksum(checksumHeaderName) + } + for _, partNumber := range completedPartNumbers { + if provided[partNumber] == "" { + return nil, nil, s3err.ErrInvalidRequest + } + raw, _, decodeErr := decodePartChecksum(partNumber, partEntries[partNumber], checksumHeaderName) + if decodeErr != nil { + glog.Errorf("completeMultipartUpload: %v", decodeErr) + return nil, nil, s3err.ErrInvalidPart + } + if provided[partNumber] != base64.StdEncoding.EncodeToString(raw) { + return nil, nil, s3err.ErrBadDigest + } + } + var checksumErr error if checksumType == s3_constants.ChecksumTypeFullObject { checksumValue, checksumErr = computeFullObjectChecksum(checksumHeaderName, partEntries, completedPartNumbers) @@ -656,7 +677,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl routeKey := s3a.objectRouteKey(*input.Bucket, *input.Key) completionBody := func() s3err.ErrorCode { var prepCode s3err.ErrorCode - completionState, output, prepCode = s3a.prepareMultipartCompletionState(r, input, uploadDirectory, entryName, dirName, completedPartNumbers, completedPartMap, maxPartNo) + completionState, output, prepCode = s3a.prepareMultipartCompletionState(r, input, parts, uploadDirectory, entryName, dirName, completedPartNumbers, completedPartMap, maxPartNo) if prepCode != s3err.ErrNone || output != nil { return prepCode } diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index 9a7fe7b8e..367e87706 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -645,6 +645,7 @@ type CompleteMultipartUpload struct { type CompletedPart struct { ETag string PartNumber int + ChecksumResult } // handleSSES3MultipartHeaders handles SSE-S3 multipart upload header setup to reduce nesting complexity diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 5f8510c22..30a9cfcf9 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -1225,6 +1225,22 @@ func (c *ChecksumResult) SetChecksum(headerName, value string) { } } +func (c *ChecksumResult) GetChecksum(headerName string) string { + switch headerName { + case s3_constants.AmzChecksumCRC32: + return c.ChecksumCRC32 + case s3_constants.AmzChecksumCRC32C: + return c.ChecksumCRC32C + case s3_constants.AmzChecksumCRC64NVME: + return c.ChecksumCRC64NVME + case s3_constants.AmzChecksumSHA1: + return c.ChecksumSHA1 + case s3_constants.AmzChecksumSHA256: + return c.ChecksumSHA256 + } + return "" +} + // lookupHeaderOrQuery returns the value of an x-amz-* parameter, checking the // request headers first and falling back to the pre-parsed query values. AWS // SDK presigners hoist headers such as x-amz-sdk-checksum-algorithm into the