diff --git a/backend/posix/posix.go b/backend/posix/posix.go index a0b14656..d05e18c2 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1661,9 +1661,8 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C // The checksum algorithm should default to CRC64NVME // just for data integrity. It isn't going to be saved // in the final object metadata - checksumAlgorithm := types.ChecksumAlgorithmCrc64nvme - if checksums.Algorithm != "" { - checksumAlgorithm = checksums.Algorithm + if checksums.Algorithm == "" { + checksums.Algorithm = types.ChecksumAlgorithmCrc64nvme } // ChecksumType should be the same as specified on CreateMultipartUpload @@ -1676,13 +1675,11 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C return res, "", s3err.GetChecksumTypeMismatchOnMpErr(checksumType) } + mpChecksumType := checksums.Type + // The checksum type should default to FULL_OBJECT(crc64nvme) - checksumType := checksums.Type if checksums.Type == "" { - // do not modify checksums.Type to further not save the checksum - // in the final object. As if no checksum has been specified on mp - // creation, the final object shouldn't contain the checksum metadata - checksumType = types.ChecksumTypeFullObject + checksums.Type = types.ChecksumTypeFullObject } // check all parts ok @@ -1690,9 +1687,9 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C var totalsize int64 var composableCRC bool - switch checksums.Type { + switch mpChecksumType { case types.ChecksumTypeFullObject: - composableCRC = utils.IsChecksumComposable(checksumAlgorithm) + composableCRC = utils.IsChecksumComposable(checksums.Algorithm) } // The initialie values is the lower limit of partNumber: 0 @@ -1751,16 +1748,16 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C var hashRdr *utils.HashReader var compositeChecksumRdr *utils.CompositeChecksumReader - switch checksumType { + switch checksums.Type { case types.ChecksumTypeFullObject: if !composableCRC { - hashRdr, err = utils.NewHashReader(nil, "", utils.HashType(strings.ToLower(string(checksumAlgorithm)))) + hashRdr, err = utils.NewHashReader(nil, "", utils.HashType(strings.ToLower(string(checksums.Algorithm)))) if err != nil { return res, "", fmt.Errorf("initialize hash reader: %w", err) } } case types.ChecksumTypeComposite: - compositeChecksumRdr, err = utils.NewCompositeChecksumReader(utils.HashType(strings.ToLower(string(checksumAlgorithm)))) + compositeChecksumRdr, err = utils.NewCompositeChecksumReader(utils.HashType(strings.ToLower(string(checksums.Algorithm)))) if err != nil { return res, "", fmt.Errorf("initialize composite checksum reader: %w", err) } @@ -1791,15 +1788,15 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C } var rdr io.Reader = pf - switch checksumType { + switch checksums.Type { case types.ChecksumTypeFullObject: if composableCRC { if i == 0 { - composableCsum = getPartChecksum(checksumAlgorithm, part) + composableCsum = getPartChecksum(checksums.Algorithm, part) break } - composableCsum, err = utils.AddCRCChecksum(checksumAlgorithm, - composableCsum, getPartChecksum(checksumAlgorithm, part), + composableCsum, err = utils.AddCRCChecksum(checksums.Algorithm, + composableCsum, getPartChecksum(checksums.Algorithm, part), pfi.Size()) if err != nil { pf.Close() @@ -1811,7 +1808,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C hashRdr.SetReader(rdr) rdr = hashRdr case types.ChecksumTypeComposite: - err := compositeChecksumRdr.Process(getPartChecksum(checksumAlgorithm, part)) + err := compositeChecksumRdr.Process(getPartChecksum(checksums.Algorithm, part)) if err != nil { pf.Close() return res, "", fmt.Errorf("process %v part checksum: %w", @@ -1924,76 +1921,60 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C var sha256 *string var crc64nvme *string - // Calculate, compare with the provided checksum and store them - if checksums.Type != "" { - checksum := s3response.Checksum{ - Algorithm: checksumAlgorithm, - Type: checksums.Type, + var value string + switch checksums.Type { + case types.ChecksumTypeComposite: + value = fmt.Sprintf("%s-%v", compositeChecksumRdr.Sum(), len(parts)) + case types.ChecksumTypeFullObject: + if !composableCRC { + value = hashRdr.Sum() + } else { + value = composableCsum + } + } + + var gotSum *string + + switch checksums.Algorithm { + case types.ChecksumAlgorithmCrc32: + gotSum = input.ChecksumCRC32 + checksums.CRC32 = &value + crc32 = &value + case types.ChecksumAlgorithmCrc32c: + gotSum = input.ChecksumCRC32C + checksums.CRC32C = &value + crc32c = &value + case types.ChecksumAlgorithmSha1: + gotSum = input.ChecksumSHA1 + checksums.SHA1 = &value + sha1 = &value + case types.ChecksumAlgorithmSha256: + gotSum = input.ChecksumSHA256 + checksums.SHA256 = &value + sha256 = &value + case types.ChecksumAlgorithmCrc64nvme: + gotSum = input.ChecksumCRC64NVME + checksums.CRC64NVME = &value + crc64nvme = &value + } + + // Check if the provided checksum and the calculated one are the same + if mpChecksumType != "" && gotSum != nil { + s := *gotSum + if checksums.Type == types.ChecksumTypeComposite && !strings.Contains(s, "-") { + // if number of parts is not specified in the final checksum + // make sure to add, to not fail in the final comparison + s = fmt.Sprintf("%s-%v", s, len(parts)) } - var sum string - switch checksums.Type { - case types.ChecksumTypeComposite: - sum = fmt.Sprintf("%s-%v", compositeChecksumRdr.Sum(), len(parts)) - case types.ChecksumTypeFullObject: - if !composableCRC { - sum = hashRdr.Sum() - } else { - sum = composableCsum - } + if s != value { + return res, "", s3err.GetChecksumBadDigestErr(checksums.Algorithm) } + } - var gotSum *string - - switch checksumAlgorithm { - case types.ChecksumAlgorithmCrc32: - gotSum = input.ChecksumCRC32 - checksum.CRC32 = &sum - crc32 = &sum - case types.ChecksumAlgorithmCrc32c: - gotSum = input.ChecksumCRC32C - checksum.CRC32C = &sum - crc32c = &sum - case types.ChecksumAlgorithmSha1: - gotSum = input.ChecksumSHA1 - checksum.SHA1 = &sum - sha1 = &sum - case types.ChecksumAlgorithmSha256: - gotSum = input.ChecksumSHA256 - checksum.SHA256 = &sum - sha256 = &sum - case types.ChecksumAlgorithmCrc64nvme: - gotSum = input.ChecksumCRC64NVME - checksum.CRC64NVME = &sum - crc64nvme = &sum - } - - // Check if the provided checksum and the calculated one are the same - if gotSum != nil { - s := *gotSum - if checksums.Type == types.ChecksumTypeComposite && !strings.Contains(s, "-") { - // if number of parts is not specified in the final checksum - // make sure to add, to not fail in the final comparison - s = fmt.Sprintf("%s-%v", s, len(parts)) - } - - if s != sum { - return res, "", s3err.GetChecksumBadDigestErr(checksumAlgorithm) - } - } - - err := p.storeChecksums(f.File(), bucket, object, checksum) - if err != nil { - return res, "", fmt.Errorf("store object checksum: %w", err) - } - } else { - // in this case no checksum has been specified on mp creation - // and the complete request checksum is defaulted to crc64nvem - // simply calculated the sum to further retrun in the response - if hashRdr != nil { - sum := hashRdr.Sum() - crc64nvme = &sum - } + err = p.storeChecksums(f.File(), bucket, object, checksums) + if err != nil { + return res, "", fmt.Errorf("store object checksum: %w", err) } // load and set retention @@ -2036,7 +2017,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C ChecksumSHA1: sha1, ChecksumSHA256: sha256, ChecksumCRC64NVME: crc64nvme, - ChecksumType: &checksumType, + ChecksumType: &checksums.Type, }, versionID, nil } diff --git a/tests/integration/CompleteMultipartUpload.go b/tests/integration/CompleteMultipartUpload.go index 18643946..e086703a 100644 --- a/tests/integration/CompleteMultipartUpload.go +++ b/tests/integration/CompleteMultipartUpload.go @@ -1111,6 +1111,24 @@ func CompleteMultipartUpload_should_ignore_the_final_checksum(s *S3Conf) error { return fmt.Errorf("expected non nil crc64nvme checksum") } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if out.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the final object checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, out.ChecksumType) + } + if out.ChecksumCRC64NVME == nil { + return fmt.Errorf("expected non nil crc64nvme checksum") + } + return nil }) }