diff --git a/backend/posix/posix.go b/backend/posix/posix.go index f192b13a..a77adf73 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -2401,7 +2401,8 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. // If checksum isn't provided for the part, // but it has been provided on mp initalization - if hashRdr == nil && chErr == nil && checksums.Algorithm != "" { + // and checksum type is 'COMPOSITE', return mismatch error + if hashRdr == nil && chErr == nil && checksums.Type == types.ChecksumTypeComposite { return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithm, "null") } @@ -2414,6 +2415,18 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. } } + // if no checksum algorithm or precalculated checksum is + // provided, but one has been on multipart upload initialization, + // anyways calculate and store the uploaded part checksum + if hashRdr == nil && checksums.Algorithm != "" { + hashRdr, err = utils.NewHashReader(tr, "", utils.HashType(strings.ToLower(string(checksums.Algorithm)))) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + _, err = io.Copy(f, tr) if err != nil { if errors.Is(err, syscall.EDQUOT) { diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 289ab39f..b698840d 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -353,6 +353,9 @@ func TestUploadPart(s *S3Conf) { UploadPart_checksum_algorithm_mistmatch_on_initialization(s) UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s) UploadPart_incorrect_checksums(s) + UploadPart_no_checksum_with_full_object_checksum_type(s) + UploadPart_no_checksum_with_composite_checksum_type(s) + UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s) UploadPart_with_checksums_success(s) } UploadPart_success(s) @@ -1118,6 +1121,9 @@ func GetIntTests() IntTests { "UploadPart_checksum_algorithm_mistmatch_on_initialization": UploadPart_checksum_algorithm_mistmatch_on_initialization, "UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value": UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value, "UploadPart_incorrect_checksums": UploadPart_incorrect_checksums, + "UploadPart_no_checksum_with_full_object_checksum_type": UploadPart_no_checksum_with_full_object_checksum_type, + "UploadPart_no_checksum_with_composite_checksum_type": UploadPart_no_checksum_with_composite_checksum_type, + "UploadPart_should_calculate_checksum_if_only_algorithm_is_provided": UploadPart_should_calculate_checksum_if_only_algorithm_is_provided, "UploadPart_with_checksums_success": UploadPart_with_checksums_success, "UploadPart_success": UploadPart_success, "UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 706160bc..c868ce00 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -19,11 +19,16 @@ import ( "context" "crypto/rand" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/xml" "errors" "fmt" + "hash" + "hash/crc32" + "hash/crc64" "io" + "math/bits" "net/http" "net/url" "regexp" @@ -8648,6 +8653,165 @@ func UploadPart_incorrect_checksums(s *S3Conf) error { }) } +func UploadPart_no_checksum_with_full_object_checksum_type(s *S3Conf) error { + testName := "UploadPart_no_checksum_with_full_object_checksum_type" + return actionHandler(s, testName, func(_ *s3.Client, bucket string) error { + customClient := s3.NewFromConfig(s.Config(), func(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset + }) + obj := "my-obj" + + for _, algo := range []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc32, + types.ChecksumAlgorithmCrc32c, + types.ChecksumAlgorithmCrc64nvme, + } { + mp, err := createMp(customClient, bucket, obj, withChecksum(algo), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + var hashRdr hash.Hash + + switch algo { + case types.ChecksumAlgorithmCrc32: + hashRdr = crc32.NewIEEE() + case types.ChecksumAlgorithmCrc32c: + hashRdr = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case types.ChecksumAlgorithmCrc64nvme: + hashRdr = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))) + default: + return fmt.Errorf("invalid checksum algorithm provided: %s", algo) + } + + partBuffer := make([]byte, 5*1024*1024) + rand.Read(partBuffer) + hashRdr.Write(partBuffer) + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := customClient.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + Body: bytes.NewReader(partBuffer), + PartNumber: &partNumber, + }) + cancel() + if err != nil { + return err + } + + csum := base64.StdEncoding.EncodeToString(hashRdr.Sum(nil)) + + switch algo { + case types.ChecksumAlgorithmCrc32: + if getString(res.ChecksumCRC32) != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", algo, csum, getString(res.ChecksumCRC32)) + } + case types.ChecksumAlgorithmCrc32c: + if getString(res.ChecksumCRC32C) != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", algo, csum, getString(res.ChecksumCRC32C)) + } + case types.ChecksumAlgorithmCrc64nvme: + if getString(res.ChecksumCRC64NVME) != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", algo, csum, getString(res.ChecksumCRC64NVME)) + } + } + } + return nil + }) +} + +func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error { + testName := "UploadPart_no_checksum_with_composite_checksum_type" + return actionHandler(s, testName, func(_ *s3.Client, bucket string) error { + customClient := s3.NewFromConfig(s.Config(), func(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset + }) + obj := "my-obj" + + for _, algo := range []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc32, + types.ChecksumAlgorithmCrc32c, + types.ChecksumAlgorithmSha1, + types.ChecksumAlgorithmSha256, + } { + mp, err := createMp(customClient, bucket, obj, withChecksum(algo), withChecksumType(types.ChecksumTypeComposite)) + if err != nil { + return err + } + _, _, err = uploadParts(customClient, 10, 1, bucket, obj, *mp.UploadId) + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr(algo, "null")); err != nil { + return err + } + } + return nil + }) +} + +func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Conf) error { + testName := "UploadPart_should_calculate_checksum_if_only_algorithm_is_provided" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + customClient := s3.NewFromConfig(s.Config(), func(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset + }) + obj := "my-obj" + + for _, test := range []struct { + chType types.ChecksumType + chAlgo types.ChecksumAlgorithm + }{ + {types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc32}, + {types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc32c}, + {types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc64nvme}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256}, + } { + mp, err := createMp(customClient, bucket, obj, withChecksum(test.chAlgo), withChecksumType(test.chType)) + if err != nil { + return err + } + + parts, csum, err := uploadParts(customClient, 5*1024*1024, 1, bucket, obj, *mp.UploadId, withChecksum(test.chAlgo)) + if err != nil { + return err + } + + if len(parts) != 1 { + return fmt.Errorf("expected 1 uploaded part, instaed got %d", len(parts)) + } + + part := parts[0] + switch test.chAlgo { + case types.ChecksumAlgorithmCrc32: + if getString(part.ChecksumCRC32) != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32)) + } + case types.ChecksumAlgorithmCrc32c: + if getString(part.ChecksumCRC32C) != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32C)) + } + case types.ChecksumAlgorithmCrc64nvme: + if getString(part.ChecksumCRC64NVME) != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC64NVME)) + } + case types.ChecksumAlgorithmSha1: + if getString(part.ChecksumSHA1) != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA1)) + } + case types.ChecksumAlgorithmSha256: + if getString(part.ChecksumSHA256) != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA256)) + } + } + } + return nil + }) +} + func UploadPart_with_checksums_success(s *S3Conf) error { testName := "UploadPart_with_checksums_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {