From 695675755776de4bba5ee5a581e75e3670ad71fb Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 13 Jan 2025 22:28:46 +0400 Subject: [PATCH] feat: Integrates object integrity checksums(CRC32, CRC32C, SHA1, SHA256) into the gateway --- backend/azure/azure.go | 26 +- backend/backend.go | 12 +- backend/posix/posix.go | 805 ++++++++++++-- backend/s3proxy/s3.go | 16 +- s3api/controllers/backend_moq_test.go | 12 +- s3api/controllers/base.go | 260 ++++- s3api/controllers/base_test.go | 92 +- s3api/middlewares/md5.go | 2 +- s3api/utils/auth-reader.go | 2 +- s3api/utils/csum-reader.go | 75 +- s3api/utils/utils.go | 76 +- s3api/utils/utils_test.go | 153 +++ s3err/s3err.go | 49 + s3response/s3response.go | 78 +- tests/integration/group-tests.go | 37 +- tests/integration/tests.go | 1462 +++++++++++++++++++++++++ tests/integration/utils.go | 101 +- 17 files changed, 3072 insertions(+), 186 deletions(-) diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 27a433d..bef5dd7 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -905,9 +905,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult } // Each part is translated into an uncommitted block in a newly created blob in staging area -func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) { +func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil { - return "", err + return nil, err } // TODO: request streamable version of StageBlock() @@ -916,32 +916,34 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (eta // the body in memory to create an io.ReadSeekCloser rdr, err := getReadSeekCloser(input.Body) if err != nil { - return "", err + return nil, err } client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) if err != nil { - return "", err + return nil, err } // block id serves as etag here - etag = blockIDInt32ToBase64(*input.PartNumber) + etag := blockIDInt32ToBase64(*input.PartNumber) _, err = client.StageBlock(ctx, etag, rdr, nil) if err != nil { - return "", parseMpError(err) + return nil, parseMpError(err) } - return etag, nil + return &s3.UploadPartOutput{ + ETag: &etag, + }, nil } -func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) if err != nil { - return s3response.CopyObjectResult{}, nil + return s3response.CopyPartResult{}, nil } if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil { - return s3response.CopyObjectResult{}, err + return s3response.CopyPartResult{}, err } eTag := blockIDInt32ToBase64(*input.PartNumber) @@ -949,10 +951,10 @@ func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInp //TODO: the action returns not implemented on azurite, maybe in production this will work? _, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil) if err != nil { - return s3response.CopyObjectResult{}, parseMpError(err) + return s3response.CopyPartResult{}, parseMpError(err) } - return s3response.CopyObjectResult{}, nil + return s3response.CopyPartResult{}, nil } // Lists all uncommitted parts from the blob diff --git a/backend/backend.go b/backend/backend.go index 1141d64..2f4e67a 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -53,8 +53,8 @@ type Backend interface { AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) - UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) - UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) + UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) + UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) // standard object operations PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) @@ -166,11 +166,11 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) { return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) { - return "", s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { diff --git a/backend/posix/posix.go b/backend/posix/posix.go index ee1c167..1ded60e 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "io/fs" + "net/http" "os" "path/filepath" "sort" @@ -39,6 +40,7 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/backend/meta" + "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -87,6 +89,7 @@ const ( aclkey = "acl" ownershipkey = "ownership" etagkey = "etag" + checksumsKey = "checksums" policykey = "policy" bucketLockKey = "bucket-lock" objectRetentionKey = "object-retention" @@ -1308,6 +1311,21 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa } } + // Set object checksum algorithm + if mpu.ChecksumAlgorithm != "" { + err := p.storeChecksums(nil, bucket, filepath.Join(objdir, uploadID), s3response.Checksum{ + Algorithms: []types.ChecksumAlgorithm{ + mpu.ChecksumAlgorithm, + }, + }) + if err != nil { + // cleanup object if returning error + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("store mp checksum algorithm: %w", err) + } + } + return s3response.InitiateMultipartUploadResult{ Bucket: bucket, Key: object, @@ -1373,6 +1391,28 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + checksums, err := p.retreiveChecksums(nil, bucket, filepath.Join(objdir, uploadID)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get mp checksums: %w", err) + } + + // if len(checksums.Algorithms) != 0 { + // algorithm := checksums.Algorithms[0] + + // if input.ChecksumCRC32 != nil && algorithm != types.ChecksumAlgorithmCrc32 { + // return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32") + // } + // if input.ChecksumCRC32C != nil && algorithm != types.ChecksumAlgorithmCrc32c { + // return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32c") + // } + // if input.ChecksumSHA1 != nil && algorithm != types.ChecksumAlgorithmSha1 { + // return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-sha1") + // } + // if input.ChecksumSHA256 != nil && algorithm != types.ChecksumAlgorithmSha256 { + // return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-sha256") + // } + // } + // check all parts ok last := len(parts) - 1 var totalsize int64 @@ -1403,6 +1443,25 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM if parts[i].ETag == nil || etag != *parts[i].ETag { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } + + partChecksum, err := p.retreiveChecksums(nil, bucket, partObjPath) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get part checksum: %w", err) + } + + // If checksum has been provided on mp initalization + err = validatePartChecksum(partChecksum, part) + if err != nil { + return nil, err + } + } + + var hashRdr *utils.HashReader + if len(checksums.Algorithms) != 0 { + hashRdr, err = utils.NewHashReader(nil, "", utils.HashType(strings.ToLower(string(checksums.Algorithms[0])))) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } } f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, @@ -1426,7 +1485,14 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM if err != nil { return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err) } - _, err = io.Copy(f.File(), pf) + + var rdr io.Reader = pf + if hashRdr != nil { + hashRdr.SetReader(rdr) + rdr = hashRdr + } + + _, err = io.Copy(f.File(), rdr) pf.Close() if err != nil { if errors.Is(err, syscall.EDQUOT) { @@ -1524,6 +1590,53 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } } + var crc32 *string + var crc32c *string + var sha1 *string + var sha256 *string + + // set checksum + if hashRdr != nil { + algo := checksums.Algorithms[0] + checksum := s3response.Checksum{ + Algorithms: []types.ChecksumAlgorithm{ + algo, + }, + } + + sum := hashRdr.Sum() + switch hashRdr.Type() { + case utils.HashTypeCRC32: + if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 != sum { + return nil, s3err.GetChecksumBadDigestErr(algo) + } + checksum.CRC32 = &sum + crc32 = &sum + case utils.HashTypeCRC32C: + if input.ChecksumCRC32C != nil && *input.ChecksumCRC32C != sum { + return nil, s3err.GetChecksumBadDigestErr(algo) + } + checksum.CRC32C = &sum + crc32c = &sum + case utils.HashTypeSha1: + if input.ChecksumSHA1 != nil && *input.ChecksumSHA1 != sum { + return nil, s3err.GetChecksumBadDigestErr(algo) + } + checksum.SHA1 = &sum + sha1 = &sum + case utils.HashTypeSha256: + if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 != sum { + return nil, s3err.GetChecksumBadDigestErr(algo) + } + checksum.SHA256 = &sum + sha256 = &sum + } + err := p.storeChecksums(f.File(), bucket, object, checksum) + if err != nil { + return nil, fmt.Errorf("store object checksum: %w", err) + } + } + // load and set retention ret, err := p.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -1556,13 +1669,130 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM os.Remove(filepath.Join(bucket, objdir)) return &s3.CompleteMultipartUploadOutput{ - Bucket: &bucket, - ETag: &s3MD5, - Key: &object, - VersionId: &versionID, + Bucket: &bucket, + ETag: &s3MD5, + Key: &object, + VersionId: &versionID, + ChecksumCRC32: crc32, + ChecksumCRC32C: crc32c, + ChecksumSHA1: sha1, + ChecksumSHA256: sha256, }, nil } +func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart) error { + n := numberOfChecksums(part) + + if len(checksum.Algorithms) != 0 { + algo := checksum.Algorithms[0] + if n == 0 { + return s3err.APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The upload was created using a %v checksum. The complete request must include the checksum for each part. It was missing for part %v in the request.", strings.ToLower(string(algo)), *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + if n > 1 { + return s3err.GetAPIError(s3err.ErrInvalidChecksumPart) + } + + if part.ChecksumCRC32 != nil { + if ok := utils.IsValidChecksum(*part.ChecksumCRC32, types.ChecksumAlgorithmCrc32); !ok { + return s3err.GetAPIError(s3err.ErrInvalidChecksumPart) + } + + if *part.ChecksumCRC32 != getString(checksum.CRC32) { + if algo == types.ChecksumAlgorithmCrc32 { + return s3err.GetAPIError(s3err.ErrInvalidPart) + } else { + return s3err.APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The crc32 you specified for part %v did not match what we received.", *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + } + if part.ChecksumCRC32C != nil { + if ok := utils.IsValidChecksum(*part.ChecksumCRC32C, types.ChecksumAlgorithmCrc32c); !ok { + return s3err.GetAPIError(s3err.ErrInvalidChecksumPart) + } + + if *part.ChecksumCRC32C != getString(checksum.CRC32C) { + if algo == types.ChecksumAlgorithmCrc32c { + return s3err.GetAPIError(s3err.ErrInvalidPart) + } else { + return s3err.APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The crc32c you specified for part %v did not match what we received.", *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + } + if part.ChecksumSHA1 != nil { + if ok := utils.IsValidChecksum(*part.ChecksumSHA1, types.ChecksumAlgorithmSha1); !ok { + return s3err.GetAPIError(s3err.ErrInvalidChecksumPart) + } + + if *part.ChecksumSHA1 != getString(checksum.SHA1) { + if algo == types.ChecksumAlgorithmSha1 { + return s3err.GetAPIError(s3err.ErrInvalidPart) + } else { + return s3err.APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The sha1 you specified for part %v did not match what we received.", *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + } + if part.ChecksumSHA256 != nil { + if ok := utils.IsValidChecksum(*part.ChecksumSHA256, types.ChecksumAlgorithmSha256); !ok { + return s3err.GetAPIError(s3err.ErrInvalidChecksumPart) + } + + if *part.ChecksumSHA256 != getString(checksum.SHA256) { + if algo == types.ChecksumAlgorithmSha256 { + return s3err.GetAPIError(s3err.ErrInvalidPart) + } else { + return s3err.APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The sha256 you specified for part %v did not match what we received.", *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + } + + return nil + } + + if n != 0 { + return s3err.GetAPIError(s3err.ErrInvalidPart) + } + + return nil +} + +func numberOfChecksums(part types.CompletedPart) int { + counter := 0 + if getString(part.ChecksumCRC32) != "" { + counter++ + } + if getString(part.ChecksumCRC32C) != "" { + counter++ + } + if getString(part.ChecksumSHA1) != "" { + counter++ + } + if getString(part.ChecksumSHA256) != "" { + counter++ + } + + return counter +} + func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) { sum := sha256.Sum256([]byte(object)) objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) @@ -1752,11 +1982,23 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl if keyMarkerInd == -1 && objectName == keyMarker { keyMarkerInd = len(uploads) } + + checksum, err := p.retreiveChecksums(nil, bucket, filepath.Join(metaTmpMultipartDir, obj.Name(), uploadID)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return lmu, fmt.Errorf("get mp checksum: %w", err) + } + + var algo types.ChecksumAlgorithm + if len(checksum.Algorithms) != 0 { + algo = checksum.Algorithms[0] + } + uploads = append(uploads, s3response.Upload{ - Key: objectName, - UploadID: uploadID, - StorageClass: types.StorageClassStandard, - Initiated: fi.ModTime(), + Key: objectName, + UploadID: uploadID, + StorageClass: types.StorageClassStandard, + Initiated: fi.ModTime(), + ChecksumAlgorithm: algo, }) } } @@ -1875,6 +2117,16 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon return lpr, fmt.Errorf("readdir upload: %w", err) } + checksum, err := p.retreiveChecksums(nil, tmpdir, uploadID) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return lpr, fmt.Errorf("get mp checksum: %w", err) + } + + var algo types.ChecksumAlgorithm + if len(checksum.Algorithms) != 0 { + algo = checksum.Algorithms[0] + } + var parts []s3response.Part for _, e := range ents { pn, err := strconv.Atoi(e.Name()) @@ -1893,16 +2145,25 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon etag = "" } + checksum, err := p.retreiveChecksums(nil, bucket, partPath) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + continue + } + fi, err := os.Lstat(filepath.Join(bucket, partPath)) if err != nil { continue } parts = append(parts, s3response.Part{ - PartNumber: pn, - ETag: etag, - LastModified: fi.ModTime(), - Size: fi.Size(), + PartNumber: pn, + ETag: etag, + LastModified: fi.ModTime(), + Size: fi.Size(), + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, }) } @@ -1934,20 +2195,21 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon Parts: parts, UploadID: uploadID, StorageClass: types.StorageClassStandard, + ChecksumAlgorithm: algo, }, nil } -func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (string, error) { +func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} } if input.Bucket == nil { - return "", s3err.GetAPIError(s3err.ErrInvalidBucketName) + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) } if input.Key == nil { - return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } bucket := *input.Bucket @@ -1962,79 +2224,185 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (stri _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return "", s3err.GetAPIError(s3err.ErrNoSuchBucket) + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return "", fmt.Errorf("stat bucket: %w", err) + return nil, fmt.Errorf("stat bucket: %w", err) } sum := sha256.Sum256([]byte(object)) objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + mpPath := filepath.Join(objdir, uploadID) - _, err = os.Stat(filepath.Join(bucket, objdir, uploadID)) + _, err = os.Stat(filepath.Join(bucket, mpPath)) if errors.Is(err, fs.ErrNotExist) { - return "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) } if err != nil { - return "", fmt.Errorf("stat uploadid: %w", err) + return nil, fmt.Errorf("stat uploadid: %w", err) } - partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part)) + partPath := filepath.Join(mpPath, fmt.Sprintf("%v", *part)) f, err := p.openTmpFile(filepath.Join(bucket, objdir), bucket, partPath, length, acct, doFalloc) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", fmt.Errorf("open temp file: %w", err) + return nil, fmt.Errorf("open temp file: %w", err) } defer f.cleanup() hash := md5.New() tr := io.TeeReader(r, hash) + + var hashRdr *utils.HashReader + if input.ChecksumCRC32 != nil { + hashRdr, err = utils.NewHashReader(tr, *input.ChecksumCRC32, utils.HashTypeCRC32) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + if input.ChecksumCRC32C != nil { + hashRdr, err = utils.NewHashReader(tr, *input.ChecksumCRC32C, utils.HashTypeCRC32C) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + if input.ChecksumSHA1 != nil { + hashRdr, err = utils.NewHashReader(tr, *input.ChecksumSHA1, utils.HashTypeSha1) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + if input.ChecksumSHA256 != nil { + hashRdr, err = utils.NewHashReader(tr, *input.ChecksumSHA256, utils.HashTypeSha256) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + + // If only the checksum algorithm is provided register + // a new HashReader to calculate the object checksum + if hashRdr == nil && input.ChecksumAlgorithm != "" { + hashRdr, err = utils.NewHashReader(tr, "", utils.HashType(strings.ToLower(string(input.ChecksumAlgorithm)))) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + + checksums, chErr := p.retreiveChecksums(nil, bucket, mpPath) + if chErr != nil && !errors.Is(chErr, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("retreive mp checksum: %w", chErr) + } + + // If checksum isn't provided for the part, + // but it has been provided on mp initalization + if hashRdr == nil && chErr == nil && len(checksums.Algorithms) != 0 { + return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithms[0], "null") + } + + // Check if the provided checksum algorithm match + // the one specified on mp initialization + if hashRdr != nil { + algo := types.ChecksumAlgorithm(strings.ToUpper(string(hashRdr.Type()))) + if chErr != nil { + return nil, s3err.GetChecksumTypeMismatchErr("null", algo) + } + + if len(checksums.Algorithms) == 0 { + return nil, s3err.GetChecksumTypeMismatchErr("null", algo) + } + + if checksums.Algorithms[0] != algo { + return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithms[0], algo) + } + } + _, err = io.Copy(f, tr) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", fmt.Errorf("write part data: %w", err) + return nil, fmt.Errorf("write part data: %w", err) } dataSum := hash.Sum(nil) etag := hex.EncodeToString(dataSum) err = p.meta.StoreAttribute(f.File(), bucket, partPath, etagkey, []byte(etag)) if err != nil { - return "", fmt.Errorf("set etag attr: %w", err) + return nil, fmt.Errorf("set etag attr: %w", err) + } + + res := &s3.UploadPartOutput{ + ETag: &etag, + } + + // Store the calculated checksum in the object metadata + if hashRdr != nil { + checksum := s3response.Checksum{ + Algorithms: []types.ChecksumAlgorithm{input.ChecksumAlgorithm}, + } + sum := hashRdr.Sum() + switch hashRdr.Type() { + case utils.HashTypeCRC32: + checksum.CRC32 = &sum + res.ChecksumCRC32 = &sum + case utils.HashTypeCRC32C: + checksum.CRC32C = &sum + res.ChecksumCRC32C = &sum + case utils.HashTypeSha1: + checksum.SHA1 = &sum + res.ChecksumSHA1 = &sum + case utils.HashTypeSha256: + checksum.SHA256 = &sum + res.ChecksumSHA256 = &sum + } + + err := p.storeChecksums(f.File(), bucket, partPath, checksum) + if err != nil { + return nil, fmt.Errorf("store checksum: %w", err) + } } err = f.link() if err != nil { - return "", fmt.Errorf("link object in namespace: %w", err) + return nil, fmt.Errorf("link object in namespace: %w", err) } - return etag, nil + return res, nil } -func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} } if upi.Bucket == nil { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } if upi.Key == nil { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } _, err := os.Stat(*upi.Bucket) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat bucket: %w", err) } sum := sha256.Sum256([]byte(*upi.Key)) @@ -2042,46 +2410,46 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) _, err = os.Stat(filepath.Join(*upi.Bucket, objdir, *upi.UploadId)) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchUpload) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchUpload) } if errors.Is(err, syscall.ENAMETOOLONG) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrKeyTooLong) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrKeyTooLong) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat uploadid: %w", err) } partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber)) srcBucket, srcObject, srcVersionId, err := backend.ParseCopySource(*upi.CopySource) if err != nil { - return s3response.CopyObjectResult{}, err + return s3response.CopyPartResult{}, err } _, err = os.Stat(srcBucket) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat bucket: %w", err) } vStatus, err := p.getBucketVersioningStatus(ctx, srcBucket) if err != nil { - return s3response.CopyObjectResult{}, err + return s3response.CopyPartResult{}, err } vEnabled := p.isBucketVersioningEnabled(vStatus) if srcVersionId != "" { if !p.versioningEnabled() || !vEnabled { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidVersionId) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrInvalidVersionId) } vId, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, versionIdKey) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return s3response.CopyObjectResult{}, fmt.Errorf("get src object version id: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("get src object version id: %w", err) } if string(vId) != srcVersionId { @@ -2094,20 +2462,20 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) fi, err := os.Stat(objPath) if errors.Is(err, fs.ErrNotExist) { if p.versioningEnabled() && vEnabled { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchVersion) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchVersion) } - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if errors.Is(err, syscall.ENAMETOOLONG) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrKeyTooLong) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrKeyTooLong) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat object: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat object: %w", err) } startOffset, length, err := backend.ParseRange(fi.Size(), *upi.CopySourceRange) if err != nil { - return s3response.CopyObjectResult{}, err + return s3response.CopyPartResult{}, err } if length == -1 { @@ -2115,25 +2483,25 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) } if startOffset+length > fi.Size()+1 { - return s3response.CopyObjectResult{}, backend.CreateExceedingRangeErr(fi.Size()) + return s3response.CopyPartResult{}, backend.CreateExceedingRangeErr(fi.Size()) } f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir), *upi.Bucket, partPath, length, acct, doFalloc) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return s3response.CopyObjectResult{}, fmt.Errorf("open temp file: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("open temp file: %w", err) } defer f.cleanup() srcf, err := os.Open(objPath) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("open object: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err) } defer srcf.Close() @@ -2141,35 +2509,97 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) hash := md5.New() tr := io.TeeReader(rdr, hash) + mpChecksums, err := p.retreiveChecksums(nil, *upi.Bucket, filepath.Join(objdir, *upi.UploadId)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyPartResult{}, fmt.Errorf("retreive mp checksums: %w", err) + } + + checksums, err := p.retreiveChecksums(nil, objPath, "") + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyPartResult{}, fmt.Errorf("retreive object part checksums: %w", err) + } + + var hashRdr *utils.HashReader + if len(mpChecksums.Algorithms) != 0 { + if len(checksums.Algorithms) == 0 || (mpChecksums.Algorithms[0] != checksums.Algorithms[0]) { + hashRdr, err = utils.NewHashReader(tr, "", utils.HashType(strings.ToLower(string(mpChecksums.Algorithms[0])))) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + } + _, err = io.Copy(f, tr) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + return s3response.CopyPartResult{}, fmt.Errorf("copy part data: %w", err) + } + + if len(checksums.Algorithms) != 0 { + if len(mpChecksums.Algorithms) == 0 { + checksums = s3response.Checksum{} + } else { + if hashRdr == nil { + err := p.storeChecksums(f.File(), objPath, "", checksums) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("store part checksum: %w", err) + } + } + } + } + if hashRdr != nil { + algo := types.ChecksumAlgorithm(strings.ToUpper(string(hashRdr.Type()))) + checksums = s3response.Checksum{ + Algorithms: []types.ChecksumAlgorithm{algo}, + } + + sum := hashRdr.Sum() + switch algo { + case types.ChecksumAlgorithmCrc32: + checksums.CRC32 = &sum + case types.ChecksumAlgorithmCrc32c: + checksums.CRC32C = &sum + case types.ChecksumAlgorithmSha1: + checksums.SHA1 = &sum + case types.ChecksumAlgorithmSha256: + checksums.SHA256 = &sum + } + + err := p.storeChecksums(f.File(), objPath, "", checksums) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("store part checksum: %w", err) } - return s3response.CopyObjectResult{}, fmt.Errorf("copy part data: %w", err) } dataSum := hash.Sum(nil) etag := hex.EncodeToString(dataSum) err = p.meta.StoreAttribute(f.File(), *upi.Bucket, partPath, etagkey, []byte(etag)) if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("set etag attr: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("set etag attr: %w", err) } err = f.link() if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("link object in namespace: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("link object in namespace: %w", err) } fi, err = os.Stat(filepath.Join(*upi.Bucket, partPath)) if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat part path: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat part path: %w", err) } - return s3response.CopyObjectResult{ - ETag: etag, + return s3response.CopyPartResult{ + ETag: &etag, LastModified: fi.ModTime(), CopySourceVersionId: srcVersionId, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, }, nil } @@ -2307,6 +2737,52 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons hash := md5.New() rdr := io.TeeReader(po.Body, hash) + + var hashRdr *utils.HashReader + if po.ChecksumCRC32 != nil { + hashRdr, err = utils.NewHashReader(rdr, *po.ChecksumCRC32, utils.HashTypeCRC32) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } + if po.ChecksumCRC32C != nil { + hashRdr, err = utils.NewHashReader(rdr, *po.ChecksumCRC32C, utils.HashTypeCRC32C) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } + if po.ChecksumSHA1 != nil { + hashRdr, err = utils.NewHashReader(rdr, *po.ChecksumSHA1, utils.HashTypeSha1) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } + if po.ChecksumSHA256 != nil { + hashRdr, err = utils.NewHashReader(rdr, *po.ChecksumSHA256, utils.HashTypeSha256) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } + + // If only the checksum algorithm is provided register + // a new HashReader to calculate the object checksum + if hashRdr == nil && po.ChecksumAlgorithm != "" { + hashRdr, err = utils.NewHashReader(rdr, "", utils.HashType(strings.ToLower(string(po.ChecksumAlgorithm)))) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } + _, err = io.Copy(f, rdr) if err != nil { if errors.Is(err, syscall.EDQUOT) { @@ -2351,6 +2827,33 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons } } + // Store the calculated checksum in the object metadata + if hashRdr != nil { + checksum := s3response.Checksum{ + Algorithms: []types.ChecksumAlgorithm{po.ChecksumAlgorithm}, + } + sum := hashRdr.Sum() + switch hashRdr.Type() { + case utils.HashTypeCRC32: + checksum.CRC32 = &sum + po.ChecksumCRC32 = &sum + case utils.HashTypeCRC32C: + checksum.CRC32C = &sum + po.ChecksumCRC32C = &sum + case utils.HashTypeSha1: + checksum.SHA1 = &sum + po.ChecksumSHA1 = &sum + case utils.HashTypeSha256: + checksum.SHA256 = &sum + po.ChecksumSHA256 = &sum + } + + err := p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) + } + } + err = p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, etagkey, []byte(etag)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) @@ -2431,8 +2934,12 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons } return s3response.PutObjectOutput{ - ETag: etag, - VersionID: versionID, + ETag: etag, + VersionID: versionID, + ChecksumCRC32: po.ChecksumCRC32, + ChecksumCRC32C: po.ChecksumCRC32C, + ChecksumSHA1: po.ChecksumSHA1, + ChecksumSHA256: po.ChecksumSHA256, }, nil } @@ -2968,6 +3475,14 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO return nil, fmt.Errorf("open object: %w", err) } + var checksums s3response.Checksum + if input.ChecksumMode == types.ChecksumModeEnabled { + checksums, err = p.retreiveChecksums(f, bucket, object) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object checksums: %w", err) + } + } + // using an os.File allows zero-copy sendfile via io.Copy(os.File, net.Conn) var body io.ReadCloser = f if startOffset != 0 || length != objSize { @@ -2988,6 +3503,10 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO StorageClass: types.StorageClassStandard, VersionId: &versionId, Body: body, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, }, nil } @@ -3161,6 +3680,14 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } } + var checksums s3response.Checksum + if input.ChecksumMode == types.ChecksumModeEnabled { + checksums, err = p.retreiveChecksums(nil, bucket, object) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object checksums: %w", err) + } + } + return &s3.HeadObjectOutput{ ContentLength: &size, ContentType: &contentType, @@ -3173,14 +3700,19 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ObjectLockRetainUntilDate: objectLockRetainUntilDate, StorageClass: types.StorageClassStandard, VersionId: &versionId, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, }, nil } func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) { data, err := p.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: input.Bucket, - Key: input.Key, - VersionId: input.VersionId, + Bucket: input.Bucket, + Key: input.Key, + VersionId: input.VersionId, + ChecksumMode: types.ChecksumModeEnabled, }) if err != nil { if errors.Is(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) && data != nil { @@ -3200,6 +3732,12 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr LastModified: data.LastModified, VersionId: data.VersionId, DeleteMarker: data.DeleteMarker, + Checksum: &types.Checksum{ + ChecksumCRC32: data.ChecksumCRC32, + ChecksumCRC32C: data.ChecksumCRC32C, + ChecksumSHA1: data.ChecksumSHA1, + ChecksumSHA256: data.ChecksumSHA256, + }, }, nil } @@ -3296,6 +3834,10 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. var etag string var version *string + var crc32 *string + var crc32c *string + var sha1 *string + var sha256 *string dstObjdPath := filepath.Join(dstBucket, dstObject) if dstObjdPath == objPath { @@ -3318,6 +3860,62 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. } } + checksums, err := p.retreiveChecksums(nil, dstBucket, dstObject) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj checksums: %w", err) + } + + if input.ChecksumAlgorithm != "" { + var algo types.ChecksumAlgorithm + if len(checksums.Algorithms) != 0 { + algo = checksums.Algorithms[0] + } + + // If a different checksum algorith is specified + // first caclculate and store the checksum + if algo != input.ChecksumAlgorithm { + f, err := os.Open(dstObjdPath) + if err != nil { + return nil, fmt.Errorf("open obj file: %w", err) + } + defer f.Close() + + hashReader, err := utils.NewHashReader(f, "", utils.HashType(strings.ToLower(string(input.ChecksumAlgorithm)))) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + _, err = hashReader.Read(nil) + if err != nil { + return nil, fmt.Errorf("read err: %w", err) + } + + checksums = s3response.Checksum{ + Algorithms: []types.ChecksumAlgorithm{input.ChecksumAlgorithm}, + } + sum := hashReader.Sum() + switch hashReader.Type() { + case utils.HashTypeCRC32: + checksums.CRC32 = &sum + crc32 = &sum + case utils.HashTypeCRC32C: + checksums.CRC32C = &sum + crc32c = &sum + case utils.HashTypeSha1: + checksums.SHA1 = &sum + sha1 = &sum + case utils.HashTypeSha256: + checksums.SHA256 = &sum + sha256 = &sum + } + + err = p.storeChecksums(f, dstBucket, dstObject, checksums) + if err != nil { + return nil, fmt.Errorf("store checksum: %w", err) + } + } + } + b, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, etagkey) etag = string(b) vId, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, versionIdKey) @@ -3327,19 +3925,40 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. version = backend.GetPtrFromString(string(vId)) } else { contentLength := fi.Size() + + checksums, err := p.retreiveChecksums(f, srcBucket, srcObject) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj checksum: %w", err) + } + + // If any checksum algorithm is provided, replace, otherwise + // use the existing one + if input.ChecksumAlgorithm != "" { + checksums.Algorithms = []types.ChecksumAlgorithm{input.ChecksumAlgorithm} + } + var chAlgorithm types.ChecksumAlgorithm + if len(checksums.Algorithms) != 0 { + chAlgorithm = checksums.Algorithms[0] + } + res, err := p.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &dstBucket, - Key: &dstObject, - Body: f, - ContentLength: &contentLength, - Metadata: input.Metadata, + Bucket: &dstBucket, + Key: &dstObject, + Body: f, + ContentLength: &contentLength, + Metadata: input.Metadata, + ChecksumAlgorithm: chAlgorithm, }) if err != nil { return nil, err } etag = res.ETag version = &res.VersionID + crc32 = res.ChecksumCRC32 + crc32c = res.ChecksumCRC32C + sha1 = res.ChecksumSHA1 + sha256 = res.ChecksumSHA256 } fi, err = os.Stat(dstObjdPath) @@ -3349,8 +3968,12 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. return &s3.CopyObjectOutput{ CopyObjectResult: &types.CopyObjectResult{ - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + ChecksumCRC32: crc32, + ChecksumCRC32C: crc32c, + ChecksumSHA1: sha1, + ChecksumSHA256: sha256, }, VersionId: version, CopySourceVersionId: &srcVersionId, @@ -3447,6 +4070,12 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { return s3response.Object{}, backend.ErrSkipObj } + // Retreive the object checksum algorithm + checksums, err := p.retreiveChecksums(nil, bucket, path) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.Object{}, backend.ErrSkipObj + } + // file object, get object info and fill out object data etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) if errors.Is(err, fs.ErrNotExist) { @@ -3472,11 +4101,12 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { mtime := fi.ModTime() return s3response.Object{ - ETag: &etag, - Key: &path, - LastModified: &mtime, - Size: &size, - StorageClass: types.ObjectStorageClassStandard, + ETag: &etag, + Key: &path, + LastModified: &mtime, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ChecksumAlgorithm: checksums.Algorithms, }, nil } } @@ -4107,6 +4737,25 @@ func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response. return buckets, nil } +func (p *Posix) storeChecksums(f *os.File, bucket, object string, chs s3response.Checksum) error { + checksums, err := json.Marshal(chs) + if err != nil { + return fmt.Errorf("parse checksum: %w", err) + } + + return p.meta.StoreAttribute(f, bucket, object, checksumsKey, checksums) +} + +func (p *Posix) retreiveChecksums(f *os.File, bucket, object string) (checksums s3response.Checksum, err error) { + checksumsAtr, err := p.meta.RetrieveAttribute(f, bucket, object, checksumsKey) + if err != nil { + return checksums, err + } + + err = json.Unmarshal(checksumsAtr, &checksums) + return checksums, err +} + func getString(str *string) string { if str == nil { return "" diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index 9f0b0b1..d3fdf34 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -332,28 +332,24 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re }, nil } -func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) { +func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { // streaming backend is not seekable, // use unsigned payload for streaming ops output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions( v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware, )) - if err != nil { - return "", handleError(err) - } - - return *output.ETag, nil + return output, handleError(err) } -func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { output, err := s.client.UploadPartCopy(ctx, input) if err != nil { - return s3response.CopyObjectResult{}, handleError(err) + return s3response.CopyPartResult{}, handleError(err) } - return s3response.CopyObjectResult{ + return s3response.CopyPartResult{ LastModified: *output.CopyPartResult.LastModified, - ETag: *output.CopyPartResult.ETag, + ETag: output.CopyPartResult.ETag, }, nil } diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 06124fa..b29b281 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -170,10 +170,10 @@ var _ backend.Backend = &BackendMock{} // StringFunc: func() string { // panic("mock out the String method") // }, -// UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) { +// UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) { // panic("mock out the UploadPart method") // }, -// UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +// UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { // panic("mock out the UploadPartCopy method") // }, // } @@ -331,10 +331,10 @@ type BackendMock struct { StringFunc func() string // UploadPartFunc mocks the UploadPart method. - UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) + UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) // UploadPartCopyFunc mocks the UploadPartCopy method. - UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) + UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) // calls tracks calls to the methods. calls struct { @@ -2620,7 +2620,7 @@ func (mock *BackendMock) StringCalls() []struct { } // UploadPart calls UploadPartFunc. -func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) { +func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) { if mock.UploadPartFunc == nil { panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called") } @@ -2656,7 +2656,7 @@ func (mock *BackendMock) UploadPartCalls() []struct { } // UploadPartCopy calls UploadPartCopyFunc. -func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { if mock.UploadPartCopyFunc == nil { panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called") } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 02947bf..4c80502 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -485,12 +485,27 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { }) } + checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + if c.debug { + log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode) + } + return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionGetObject, + BucketOwner: parsedAcl.Owner, + }) + } + ctx.Locals("logResBody", false) res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{ - Bucket: &bucket, - Key: &key, - Range: &acceptRange, - VersionId: &versionId, + Bucket: &bucket, + Key: &key, + Range: &acceptRange, + VersionId: &versionId, + ChecksumMode: checksumMode, }) if err != nil { if res != nil { @@ -568,6 +583,30 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Value: string(res.StorageClass), }) } + if res.ChecksumCRC32 != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc32", + Value: *res.ChecksumCRC32, + }) + } + if res.ChecksumCRC32C != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc32c", + Value: *res.ChecksumCRC32C, + }) + } + if res.ChecksumSHA1 != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-sha1", + Value: *res.ChecksumSHA1, + }) + } + if res.ChecksumSHA256 != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-sha256", + Value: *res.ChecksumSHA256, + }) + } // Set x-amz-meta-... headers utils.SetMetaHeaders(ctx, res.Metadata) @@ -2007,6 +2046,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { }) } + algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + return SendResponse(ctx, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }) + } + var body io.Reader bodyi := ctx.Locals("body-reader") if bodyi != nil { @@ -2016,16 +2066,55 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } ctx.Locals("logReqBody", false) - etag, err := c.be.UploadPart(ctx.Context(), + res, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{ - Bucket: &bucket, - Key: &keyStart, - UploadId: &uploadId, - PartNumber: &partNumber, - ContentLength: &contentLength, - Body: body, + Bucket: &bucket, + Key: &keyStart, + UploadId: &uploadId, + PartNumber: &partNumber, + ContentLength: &contentLength, + Body: body, + ChecksumAlgorithm: algorithm, + ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), }) - ctx.Response().Header.Set("Etag", etag) + if err == nil { + headers := []utils.CustomHeader{} + if res.ETag != nil { + headers = append(headers, utils.CustomHeader{ + Key: "ETag", + Value: *res.ETag, + }) + } + if res.ChecksumCRC32 != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc32", + Value: *res.ChecksumCRC32, + }) + } + if res.ChecksumCRC32C != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc32c", + Value: *res.ChecksumCRC32C, + }) + } + if res.ChecksumSHA1 != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-sha1", + Value: *res.ChecksumSHA1, + }) + } + if res.ChecksumSHA256 != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-sha256", + Value: *res.ChecksumSHA256, + }) + } + + utils.SetResponseHeaders(ctx, headers) + } return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, @@ -2256,6 +2345,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { metaDirective = types.MetadataDirectiveReplace } + checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) + err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) + if err != nil { + if c.debug { + log.Printf("invalid checksum algorithm: %v", checksumAlgorithm) + } + return SendXMLResponse(ctx, nil, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.CopyObject(ctx.Context(), &s3.CopyObjectInput{ Bucket: &bucket, @@ -2269,6 +2373,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Metadata: metadata, MetadataDirective: metaDirective, StorageClass: types.StorageClass(storageClass), + ChecksumAlgorithm: checksumAlgorithm, }) if err == nil { hdrs := []utils.CustomHeader{} @@ -2368,6 +2473,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { }) } + algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + return SendResponse(ctx, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }) + } + var body io.Reader bodyi := ctx.Locals("body-reader") if bodyi != nil { @@ -2390,6 +2506,11 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ObjectLockRetainUntilDate: &objLock.RetainUntilDate, ObjectLockMode: objLock.ObjectLockMode, ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, + ChecksumAlgorithm: algorithm, + ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), }) if err != nil { return SendResponse(ctx, err, @@ -2417,6 +2538,30 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Value: res.VersionID, }) } + if getstring(res.ChecksumCRC32) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc32", + Value: getstring(res.ChecksumCRC32), + }) + } + if getstring(res.ChecksumCRC32C) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc32c", + Value: getstring(res.ChecksumCRC32C), + }) + } + if getstring(res.ChecksumSHA1) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-sha1", + Value: getstring(res.ChecksumSHA1), + }) + } + if getstring(res.ChecksumSHA256) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-sha256", + Value: getstring(res.ChecksumSHA256), + }) + } utils.SetResponseHeaders(ctx, hdrs) @@ -2934,12 +3079,27 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { }) } + checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + if c.debug { + log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode) + } + return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.HeadObject(ctx.Context(), &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &key, - PartNumber: partNumber, - VersionId: &versionId, + Bucket: &bucket, + Key: &key, + PartNumber: partNumber, + VersionId: &versionId, + ChecksumMode: checksumMode, }) if err != nil { if res != nil { @@ -3022,6 +3182,30 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: string(res.StorageClass), }) } + if res.ChecksumCRC32 != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc32", + Value: *res.ChecksumCRC32, + }) + } + if res.ChecksumCRC32C != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc32c", + Value: *res.ChecksumCRC32C, + }) + } + if res.ChecksumSHA1 != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-sha1", + Value: *res.ChecksumSHA1, + }) + } + if res.ChecksumSHA256 != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-sha256", + Value: *res.ChecksumSHA256, + }) + } contentType := getstring(res.ContentType) if contentType == "" { @@ -3232,6 +3416,20 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { }) } + _, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + if c.debug { + log.Printf("err parsing checksum headers: %v", err) + } + return SendXMLResponse(ctx, nil, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.CompleteMultipartUpload(ctx.Context(), &s3.CompleteMultipartUploadInput{ Bucket: &bucket, @@ -3240,6 +3438,10 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { MultipartUpload: &types.CompletedMultipartUpload{ Parts: data.Parts, }, + ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), }) if err == nil { if getstring(res.VersionId) != "" { @@ -3305,6 +3507,21 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { metadata := utils.GetUserMetaData(&ctx.Request().Header) + checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) + err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) + if err != nil { + if c.debug { + log.Printf("invalid checksum algorithm: %v", checksumAlgorithm) + } + return SendXMLResponse(ctx, nil, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCreateMultipartUpload, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.CreateMultipartUpload(ctx.Context(), &s3.CreateMultipartUploadInput{ Bucket: &bucket, @@ -3316,7 +3533,18 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { ObjectLockMode: objLockState.ObjectLockMode, ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus, Metadata: metadata, + ChecksumAlgorithm: checksumAlgorithm, }) + if err == nil { + if checksumAlgorithm != "" { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-checksum-algorithm", + Value: string(checksumAlgorithm), + }, + }) + } + } return SendXMLResponse(ctx, res, err, &MetaOpts{ Logger: c.logger, diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 6546237..14a5be6 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -232,6 +232,9 @@ func TestS3ApiController_GetActions(t *testing.T) { getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil) getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello") + invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil) + invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode") + tests := []struct { name string app *fiber.App @@ -329,6 +332,15 @@ func TestS3ApiController_GetActions(t *testing.T) { wantErr: false, statusCode: 200, }, + { + name: "Get-actions-get-object-invalid-checksum-mode", + app: app, + args: args{ + req: invalidChecksumMode, + }, + wantErr: false, + statusCode: 400, + }, { name: "Get-actions-get-object-success", app: app, @@ -971,14 +983,14 @@ func TestS3ApiController_PutActions(t *testing.T) { PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { return s3response.PutObjectOutput{}, nil }, - UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) { - return "hello", nil + UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + return &s3.UploadPartOutput{}, nil }, PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error { return nil }, - UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { - return s3response.CopyObjectResult{}, nil + UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { + return s3response.CopyPartResult{}, nil }, PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error { return nil @@ -1012,6 +1024,11 @@ func TestS3ApiController_PutActions(t *testing.T) { cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") + // CopyObject invalid checksum algorithm + cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") + cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") + // PutObjectAcl success aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) aclReq.Header.Set("X-Amz-Acl", "private") @@ -1033,6 +1050,40 @@ func TestS3ApiController_PutActions(t *testing.T) { invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body)) invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello") + // PutObject invalid checksum algorithm + invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") + + // PutObject invalid base64 checksum + invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64") + + // PutObject invalid crc32 + invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm") + + // PutObject invalid crc32c + invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==") + + // PutObject invalid sha1 + invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==") + + // PutObject invalid sha256 + invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm") + + // PutObject multiple checksum headers + mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=") + mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==") + + // PutObject checksum algorithm and header mismatch + checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1") + checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==") + tests := []struct { name string app *fiber.App @@ -1175,6 +1226,15 @@ func TestS3ApiController_PutActions(t *testing.T) { wantErr: false, statusCode: 200, }, + { + name: "Copy-object-invalid-checksum-algorithm", + app: app, + args: args{ + req: cpyInvChecksumAlgo, + }, + wantErr: false, + statusCode: 400, + }, { name: "Copy-object-success", app: app, @@ -1642,6 +1702,9 @@ func TestS3ApiController_HeadObject(t *testing.T) { }) appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject) + invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil) + invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode") + tests := []struct { name string app *fiber.App @@ -1658,6 +1721,15 @@ func TestS3ApiController_HeadObject(t *testing.T) { wantErr: false, statusCode: 200, }, + { + name: "Head-object-invalid-checksum-mode", + app: app, + args: args{ + req: invChecksumMode, + }, + wantErr: false, + statusCode: 400, + }, { name: "Head-object-error", app: appErr, @@ -1735,6 +1807,9 @@ func TestS3ApiController_CreateActions(t *testing.T) { }) app.Post("/:bucket/:key/*", s3ApiController.CreateActions) + invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil) + invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") + tests := []struct { name string app *fiber.App @@ -1796,6 +1871,15 @@ func TestS3ApiController_CreateActions(t *testing.T) { wantErr: false, statusCode: 200, }, + { + name: "Create-multipart-upload-invalid-checksum-algorithm", + app: app, + args: args{ + req: invChecksumAlgo, + }, + wantErr: false, + statusCode: 400, + }, { name: "Create-multipart-upload-success", app: app, diff --git a/s3api/middlewares/md5.go b/s3api/middlewares/md5.go index 8dd5740..e1b81c6 100644 --- a/s3api/middlewares/md5.go +++ b/s3api/middlewares/md5.go @@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler { } sum := md5.Sum(ctx.Body()) - calculatedSum := utils.Md5SumString(sum[:]) + calculatedSum := utils.Base64SumString(sum[:]) if incomingSum != calculatedSum { return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger}) diff --git a/s3api/utils/auth-reader.go b/s3api/utils/auth-reader.go index 2bbdce3..990e5a3 100644 --- a/s3api/utils/auth-reader.go +++ b/s3api/utils/auth-reader.go @@ -56,7 +56,7 @@ func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, de var hr *HashReader hashPayload := ctx.Get("X-Amz-Content-Sha256") if !IsSpecialPayload(hashPayload) { - hr, _ = NewHashReader(r, "", HashTypeSha256) + hr, _ = NewHashReader(r, "", HashTypeSha256Hex) } else { hr, _ = NewHashReader(r, "", HashTypeNone) } diff --git a/s3api/utils/csum-reader.go b/s3api/utils/csum-reader.go index 04ce90d..a36da38 100644 --- a/s3api/utils/csum-reader.go +++ b/s3api/utils/csum-reader.go @@ -16,13 +16,16 @@ package utils import ( "crypto/md5" + "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/hex" "errors" "hash" + "hash/crc32" "io" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/s3err" ) @@ -31,11 +34,19 @@ type HashType string const ( // HashTypeMd5 generates MD5 checksum for the data stream - HashTypeMd5 = "md5" - // HashTypeSha256 generates SHA256 checksum for the data stream - HashTypeSha256 = "sha256" + HashTypeMd5 HashType = "md5" + // HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream + HashTypeSha256 HashType = "sha256" + // HashTypeSha256Hex generates SHA256 hex encoded checksum for the data stream + HashTypeSha256Hex HashType = "sha256-hex" + // HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream + HashTypeSha1 HashType = "sha1" + // HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream + HashTypeCRC32 HashType = "crc32" + // HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream + HashTypeCRC32C HashType = "crc32c" // HashTypeNone is a no-op checksum for the data stream - HashTypeNone = "none" + HashTypeNone HashType = "none" ) // HashReader is an io.Reader that calculates the checksum @@ -62,8 +73,16 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e switch ht { case HashTypeMd5: hash = md5.New() + case HashTypeSha256Hex: + hash = sha256.New() case HashTypeSha256: hash = sha256.New() + case HashTypeSha1: + hash = sha1.New() + case HashTypeCRC32: + hash = crc32.NewIEEE() + case HashTypeCRC32C: + hash = crc32.New(crc32.MakeTable(crc32.Castagnoli)) case HashTypeNone: hash = noop{} default: @@ -88,15 +107,35 @@ func (hr *HashReader) Read(p []byte) (int, error) { if errors.Is(readerr, io.EOF) && hr.sum != "" { switch hr.hashType { case HashTypeMd5: - sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil)) + sum := hr.Sum() if sum != hr.sum { return n, s3err.GetAPIError(s3err.ErrInvalidDigest) } - case HashTypeSha256: - sum := hex.EncodeToString(hr.hash.Sum(nil)) + case HashTypeSha256Hex: + sum := hr.Sum() if sum != hr.sum { return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch) } + case HashTypeCRC32: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32) + } + case HashTypeCRC32C: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32c) + } + case HashTypeSha1: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha1) + } + case HashTypeSha256: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256) + } default: return n, errInvalidHashType } @@ -104,20 +143,36 @@ func (hr *HashReader) Read(p []byte) (int, error) { return n, readerr } +func (hr *HashReader) SetReader(r io.Reader) { + hr.r = r +} + // Sum returns the checksum hash of the data read so far func (hr *HashReader) Sum() string { switch hr.hashType { case HashTypeMd5: - return Md5SumString(hr.hash.Sum(nil)) - case HashTypeSha256: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeSha256Hex: return hex.EncodeToString(hr.hash.Sum(nil)) + case HashTypeCRC32: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeCRC32C: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeSha1: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeSha256: + return Base64SumString(hr.hash.Sum(nil)) default: return "" } } +func (hr *HashReader) Type() HashType { + return hr.hashType +} + // Md5SumString converts the hash bytes to the string checksum value -func Md5SumString(b []byte) string { +func Base64SumString(b []byte) string { return base64.StdEncoding.EncodeToString(b) } diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index fe95285..2b65e20 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -16,6 +16,7 @@ package utils import ( "bytes" + "encoding/base64" "errors" "fmt" "io" @@ -296,7 +297,9 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok { output.StorageClass = "" } - fmt.Printf("%+v\n", output) + if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok { + output.Checksum = nil + } return output } @@ -456,3 +459,74 @@ func shouldEscape(c byte) bool { return true } + +func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) { + sdkAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-sdk-checksum-algorithm")) + + err := IsChecksumAlgorithmValid(sdkAlgorithm) + if err != nil { + return "", nil, err + } + + checksums := map[types.ChecksumAlgorithm]string{ + types.ChecksumAlgorithmCrc32: ctx.Get("x-amz-checksum-crc32"), + types.ChecksumAlgorithmCrc32c: ctx.Get("x-amz-checksum-crc32c"), + types.ChecksumAlgorithmSha1: ctx.Get("x-amz-checksum-sha1"), + types.ChecksumAlgorithmSha256: ctx.Get("x-amz-checksum-sha256"), + } + + headerCtr := 0 + + for al, val := range checksums { + if val != "" && !IsValidChecksum(val, al) { + return sdkAlgorithm, checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al)))) + } + // If any other checksum value is provided, + // rather than x-amz-sdk-checksum-algorithm + if sdkAlgorithm != "" && sdkAlgorithm != al && val != "" { + return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders) + } + if val != "" { + headerCtr++ + } + + if headerCtr > 1 { + return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders) + } + } + + return sdkAlgorithm, checksums, nil +} + +var checksumLengths = map[types.ChecksumAlgorithm]int{ + types.ChecksumAlgorithmCrc32: 4, + types.ChecksumAlgorithmCrc32c: 4, + types.ChecksumAlgorithmSha1: 20, + types.ChecksumAlgorithmSha256: 32, +} + +func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool { + decoded, err := base64.StdEncoding.DecodeString(checksum) + if err != nil { + return false + } + + expectedLength, exists := checksumLengths[algorithm] + if !exists { + return false + } + + return len(decoded) == expectedLength +} + +func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error { + if alg != "" && + alg != types.ChecksumAlgorithmCrc32 && + alg != types.ChecksumAlgorithmCrc32c && + alg != types.ChecksumAlgorithmSha1 && + alg != types.ChecksumAlgorithmSha256 { + return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm) + } + + return nil +} diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 1f76759..87f38f6 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -527,3 +527,156 @@ func Test_escapePath(t *testing.T) { }) } } + +func TestIsChecksumAlgorithmValid(t *testing.T) { + type args struct { + alg types.ChecksumAlgorithm + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "empty", + args: args{ + alg: "", + }, + wantErr: false, + }, + { + name: "crc32", + args: args{ + alg: types.ChecksumAlgorithmCrc32, + }, + wantErr: false, + }, + { + name: "crc32c", + args: args{ + alg: types.ChecksumAlgorithmCrc32c, + }, + wantErr: false, + }, + { + name: "sha1", + args: args{ + alg: types.ChecksumAlgorithmSha1, + }, + wantErr: false, + }, + { + name: "sha256", + args: args{ + alg: types.ChecksumAlgorithmSha256, + }, + wantErr: false, + }, + { + name: "invalid", + args: args{ + alg: types.ChecksumAlgorithm("invalid_checksum_algorithm"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := IsChecksumAlgorithmValid(tt.args.alg); (err != nil) != tt.wantErr { + t.Errorf("IsChecksumAlgorithmValid() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestIsValidChecksum(t *testing.T) { + type args struct { + checksum string + algorithm types.ChecksumAlgorithm + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "invalid-base64", + args: args{ + checksum: "invalid_base64_string", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: false, + }, + { + name: "invalid-crc32", + args: args{ + checksum: "YXNkZmFzZGZhc2Rm", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: false, + }, + { + name: "valid-crc32", + args: args{ + checksum: "ww2FVQ==", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: true, + }, + { + name: "invalid-crc32c", + args: args{ + checksum: "Zmdoa2doZmtnZmhr", + algorithm: types.ChecksumAlgorithmCrc32c, + }, + want: false, + }, + { + name: "valid-crc32c", + args: args{ + checksum: "DOsb4w==", + algorithm: types.ChecksumAlgorithmCrc32c, + }, + want: true, + }, + { + name: "invalid-sha1", + args: args{ + checksum: "YXNkZmFzZGZhc2RmYXNkZnNhZGZzYWRm", + algorithm: types.ChecksumAlgorithmSha1, + }, + want: false, + }, + { + name: "valid-sha1", + args: args{ + checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=", + algorithm: types.ChecksumAlgorithmSha1, + }, + want: true, + }, + { + name: "invalid-sha256", + args: args{ + checksum: "Zmdoa2doZmtnZmhrYXNkZmFzZGZhc2RmZHNmYXNkZg==", + algorithm: types.ChecksumAlgorithmSha256, + }, + want: false, + }, + { + name: "valid-sha256", + args: args{ + checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=", + algorithm: types.ChecksumAlgorithmSha256, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidChecksum(tt.args.checksum, tt.args.algorithm); got != tt.want { + t.Errorf("IsValidChecksum() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/s3err/s3err.go b/s3err/s3err.go index 3db2763..9f67355 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -17,7 +17,11 @@ package s3err import ( "bytes" "encoding/xml" + "fmt" "net/http" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3/types" ) // APIError structure @@ -140,6 +144,9 @@ const ( ErrInvalidVersionId ErrNoSuchVersion ErrSuspendedVersioningNotAllowed + ErrMultipleChecksumHeaders + ErrInvalidChecksumAlgorithm + ErrInvalidChecksumPart // Non-AWS errors ErrExistingObjectIsDirectory @@ -584,6 +591,21 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.", HTTPStatusCode: http.StatusBadRequest, }, + ErrMultipleChecksumHeaders: { + Code: "InvalidRequest", + Description: "Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidChecksumAlgorithm: { + Code: "InvalidRequest", + Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidChecksumPart: { + Code: "InvalidArgument", + Description: "Invalid Base64 or multiple checksums present in request", + HTTPStatusCode: http.StatusBadRequest, + }, // non aws errors ErrExistingObjectIsDirectory: { @@ -678,3 +700,30 @@ func encodeResponse(response interface{}) []byte { e.Encode(response) return bytesBuffer.Bytes() } + +// Returns invalid checksum error with the provided header in the error description +func GetInvalidChecksumHeaderErr(header string) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("Value for %v header is invalid.", header), + HTTPStatusCode: http.StatusBadRequest, + } +} + +// Returns checksum type mismatch APIError +func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("Checksum Type mismatch occurred, expected checksum Type: %v, actual checksum Type: %v", expected, actual), + HTTPStatusCode: http.StatusBadRequest, + } +} + +// Return incorrect checksum APIError +func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError { + return APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The %v you specified did not match the calculated checksum.", strings.ToLower(string(algo))), + HTTPStatusCode: http.StatusBadRequest, + } +} diff --git a/s3response/s3response.go b/s3response/s3response.go index 310d3ef..1b0eeb8 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -29,16 +29,24 @@ const ( ) type PutObjectOutput struct { - ETag string - VersionID string + ETag string + VersionID string + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string } // Part describes part metadata. type Part struct { - PartNumber int - LastModified time.Time - ETag string - Size int64 + PartNumber int + LastModified time.Time + ETag string + Size int64 + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string } func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -59,9 +67,10 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type ListPartsResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"` - Bucket string - Key string - UploadID string `xml:"UploadId"` + Bucket string + Key string + UploadID string `xml:"UploadId"` + ChecksumAlgorithm types.ChecksumAlgorithm Initiator Initiator Owner Owner @@ -101,6 +110,7 @@ type GetObjectAttributesResponse struct { ObjectSize *int64 StorageClass types.StorageClass `xml:",omitempty"` ObjectParts *ObjectParts + Checksum *types.Checksum // Not included in the response body VersionId *string @@ -169,13 +179,14 @@ type ListObjectsV2Result struct { } type Object struct { - ETag *string - Key *string - LastModified *time.Time - Owner *types.Owner - RestoreStatus *types.RestoreStatus - Size *int64 - StorageClass types.ObjectStorageClass + ChecksumAlgorithm []types.ChecksumAlgorithm + ETag *string + Key *string + LastModified *time.Time + Owner *types.Owner + RestoreStatus *types.RestoreStatus + Size *int64 + StorageClass types.ObjectStorageClass } func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -197,12 +208,13 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error { // Upload describes in progress multipart upload type Upload struct { - Key string - UploadID string `xml:"UploadId"` - Initiator Initiator - Owner Owner - StorageClass types.StorageClass - Initiated time.Time + Key string + UploadID string `xml:"UploadId"` + Initiator Initiator + Owner Owner + StorageClass types.StorageClass + Initiated time.Time + ChecksumAlgorithm types.ChecksumAlgorithm } func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -332,6 +344,19 @@ type CopyObjectResult struct { CopySourceVersionId string `xml:"-"` } +type CopyPartResult struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"` + LastModified time.Time + ETag *string + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string + + // not included in the body + CopySourceVersionId string `xml:"-"` +} + func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type Alias CopyObjectResult aux := &struct { @@ -467,3 +492,12 @@ func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) { type ListBucketsResult struct { Buckets []Bucket } + +type Checksum struct { + Algorithms []types.ChecksumAlgorithm + + CRC32 *string + CRC32C *string + SHA1 *string + SHA256 *string +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 282da64..0a39df8 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -138,6 +138,11 @@ func TestPutObject(s *S3Conf) { PutObject_invalid_long_tags(s) PutObject_missing_object_lock_retention_config(s) PutObject_with_object_lock(s) + PutObject_checksum_algorithm_and_header_mismatch(s) + PutObject_multiple_checksum_headers(s) + PutObject_invalid_checksum_header(s) + PutObject_incorrect_checksums(s) + PutObject_checksums_success(s) PutObject_success(s) if !s.versioningEnabled { PutObject_racey_success(s) @@ -154,6 +159,8 @@ func TestHeadObject(s *S3Conf) { HeadObject_non_existing_dir_object(s) HeadObject_with_contenttype(s) HeadObject_invalid_parent_dir(s) + HeadObject_not_enabled_checksum_mode(s) + HeadObject_checksums(s) HeadObject_success(s) } @@ -164,6 +171,7 @@ func TestGetObjectAttributes(s *S3Conf) { GetObjectAttributes_invalid_parent(s) GetObjectAttributes_empty_attrs(s) GetObjectAttributes_existing_object(s) + GetObjectAttributes_checksums(s) } func TestGetObject(s *S3Conf) { @@ -173,6 +181,8 @@ func TestGetObject(s *S3Conf) { GetObject_invalid_parent(s) GetObject_with_meta(s) GetObject_large_object(s) + GetObject_not_enabled_checksum_mode(s) + GetObject_checksums(s) GetObject_success(s) GetObject_directory_success(s) GetObject_by_range_success(s) @@ -191,6 +201,7 @@ func TestListObjects(s *S3Conf) { ListObjects_max_keys_none(s) ListObjects_marker_not_from_obj_list(s) ListObjects_list_all_objs(s) + ListObjects_with_checksum(s) } func TestListObjectsV2(s *S3Conf) { @@ -203,6 +214,7 @@ func TestListObjectsV2(s *S3Conf) { ListObjectsV2_truncated_common_prefixes(s) ListObjectsV2_all_objs_max_keys(s) ListObjectsV2_list_all_objs(s) + ListObjectsV2_with_checksum(s) ListObjectsV2_invalid_parent_prefix(s) } @@ -233,6 +245,11 @@ func TestCopyObject(s *S3Conf) { CopyObject_to_itself_with_new_metadata(s) CopyObject_CopySource_starting_with_slash(s) CopyObject_non_existing_dir_object(s) + CopyObject_invalid_checksum_algorithm(s) + CopyObject_create_checksum_on_copy(s) + CopyObject_should_copy_the_existing_checksum(s) + CopyObject_should_replace_the_existing_checksum(s) + CopyObject_to_itself_by_replacing_the_checksum(s) CopyObject_success(s) } @@ -265,6 +282,8 @@ func TestCreateMultipartUpload(s *S3Conf) { CreateMultipartUpload_with_object_lock_not_enabled(s) CreateMultipartUpload_with_object_lock_invalid_retention(s) CreateMultipartUpload_past_retain_until_date(s) + CreateMultipartUpload_invalid_checksum_algorithm(s) + CreateMultipartUpload_valid_checksum_algorithm(s) CreateMultipartUpload_success(s) } @@ -273,6 +292,15 @@ func TestUploadPart(s *S3Conf) { UploadPart_invalid_part_number(s) UploadPart_non_existing_key(s) UploadPart_non_existing_mp_upload(s) + UploadPart_checksum_algorithm_and_header_mismatch(s) + UploadPart_multiple_checksum_headers(s) + UploadPart_invalid_checksum_header(s) + UploadPart_checksum_algorithm_mistmatch_on_initialization(s) + UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s) + UploadPart_required_checksum(s) + UploadPart_null_checksum(s) + UploadPart_incorrect_checksums(s) + UploadPart_with_checksums_success(s) UploadPart_success(s) } @@ -288,6 +316,9 @@ func TestUploadPartCopy(s *S3Conf) { UploadPartCopy_by_range_invalid_range(s) UploadPartCopy_greater_range_than_obj_size(s) UploadPartCopy_by_range_success(s) + UploadPartCopy_should_copy_the_checksum(s) + UploadPartCopy_should_not_copy_the_checksum(s) + UploadPartCopy_should_calculate_the_checksum(s) } func TestListParts(s *S3Conf) { @@ -296,6 +327,7 @@ func TestListParts(s *S3Conf) { ListParts_invalid_max_parts(s) ListParts_default_max_parts(s) ListParts_truncated(s) + ListParts_with_checksums(s) ListParts_success(s) } @@ -306,6 +338,7 @@ func TestListMultipartUploads(s *S3Conf) { ListMultipartUploads_max_uploads(s) ListMultipartUploads_incorrect_next_key_marker(s) ListMultipartUploads_ignore_upload_id_marker(s) + ListMultipartUploads_with_checksums(s) ListMultipartUploads_success(s) } @@ -706,7 +739,6 @@ func GetIntTests() IntTests { "ListBuckets_invalid_max_buckets": ListBuckets_invalid_max_buckets, "ListBuckets_truncated": ListBuckets_truncated, "ListBuckets_success": ListBuckets_success, - "ListBuckets_empty_success": ListBuckets_empty_success, "DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket, "DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket, "DeleteBucket_success_status_code": DeleteBucket_success_status_code, @@ -780,11 +812,9 @@ func GetIntTests() IntTests { "ListObjectsV2_truncated_common_prefixes": ListObjectsV2_truncated_common_prefixes, "ListObjectsV2_all_objs_max_keys": ListObjectsV2_all_objs_max_keys, "ListObjectsV2_list_all_objs": ListObjectsV2_list_all_objs, - "ListObjectsV2_invalid_parent_prefix": ListObjectsV2_invalid_parent_prefix, "ListObjectVersions_VD_success": ListObjectVersions_VD_success, "DeleteObject_non_existing_object": DeleteObject_non_existing_object, "DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash, - "DeleteObject_directory_not_empty": DeleteObject_directory_not_empty, "DeleteObject_name_too_long": DeleteObject_name_too_long, "DeleteObject_non_existing_dir_object": DeleteObject_non_existing_dir_object, "DeleteObject_success": DeleteObject_success, @@ -969,7 +999,6 @@ func GetIntTests() IntTests { "IAM_userplus_CreateBucket": IAM_userplus_CreateBucket, "IAM_admin_ChangeBucketOwner": IAM_admin_ChangeBucketOwner, "IAM_ChangeBucketOwner_back_to_root": IAM_ChangeBucketOwner_back_to_root, - "IAM_ListBuckets": IAM_ListBuckets, "AccessControl_default_ACL_user_access_denied": AccessControl_default_ACL_user_access_denied, "AccessControl_default_ACL_userplus_access_denied": AccessControl_default_ACL_userplus_access_denied, "AccessControl_default_ACL_admin_successful_access": AccessControl_default_ACL_admin_successful_access, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 08c7fb5..1946167 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -2925,6 +2925,218 @@ func PutObject_with_object_lock(s *S3Conf) error { return nil } +func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error { + testName := "PutObject_checksum_algorithm_and_header_mismatch" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + ChecksumCRC32C: getPtr("m0cB1Q=="), + }) + cancel() + // FIXME: The error message for PutObject is not properly serialized by the sdk + // References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921 + + // if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + // return err + // } + if err := checkSdkApiErr(err, "InvalidRequest"); err != nil { + return err + } + + return nil + }) +} + +func PutObject_multiple_checksum_headers(s *S3Conf) error { + testName := "PutObject_multiple_checksum_headers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumSHA1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), + ChecksumCRC32C: getPtr("m0cB1Q=="), + }) + cancel() + // FIXME: The error message for PutObject is not properly serialized by the sdk + // References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921 + + // if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + // return err + // } + if err := checkSdkApiErr(err, "InvalidRequest"); err != nil { + return err + } + + return nil + }) +} + +func PutObject_invalid_checksum_header(s *S3Conf) error { + testName := "PutObject_invalid_checksum_header" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, el := range []struct { + algo string + crc32 *string + crc32c *string + sha1 *string + sha256 *string + }{ + // CRC32 tests + { + algo: "crc32", + crc32: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "crc32", + crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32 + }, + // CRC32C tests + { + algo: "crc32c", + crc32c: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "crc32c", + crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c + }, + // SHA1 tests + { + algo: "sha1", + sha1: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "sha1", + sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1 + }, + // SHA256 tests + { + algo: "sha256", + sha256: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "sha256", + sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56 + }, + } { + _, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + }, s3client) + + // FIXME: The error message for PutObject is not properly serialized by the sdk + // References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921 + + // if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + // return err + // } + if err := checkSdkApiErr(err, "InvalidRequest"); err != nil { + return err + } + } + + return nil + }) +} + +func PutObject_incorrect_checksums(s *S3Conf) error { + testName := "PutObject_incorrect_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, el := range []struct { + algo types.ChecksumAlgorithm + crc32 *string + crc32c *string + sha1 *string + sha256 *string + }{ + { + algo: types.ChecksumAlgorithmCrc32, + crc32: getPtr("DUoRhQ=="), + }, + { + algo: types.ChecksumAlgorithmCrc32c, + crc32c: getPtr("yZRlqg=="), + }, + { + algo: types.ChecksumAlgorithmSha1, + sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), + }, + { + algo: types.ChecksumAlgorithmSha256, + sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="), + }, + } { + _, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + }, s3client) + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + return err + } + } + + return nil + }) +} + +func PutObject_checksums_success(s *S3Conf) error { + testName := "PutObject_checksums_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + res, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: algo, + }, s3client) + if err != nil { + return err + } + + switch algo { + case types.ChecksumAlgorithmCrc32: + if res.res.ChecksumCRC32 == nil { + return fmt.Errorf("expected non empty crc32 checksum in the response") + } + case types.ChecksumAlgorithmCrc32c: + if res.res.ChecksumCRC32C == nil { + return fmt.Errorf("expected non empty crc32c checksum in the response") + } + case types.ChecksumAlgorithmSha1: + if res.res.ChecksumSHA1 == nil { + return fmt.Errorf("expected non empty sha1 checksum in the response") + } + case types.ChecksumAlgorithmSha256: + if res.res.ChecksumSHA256 == nil { + return fmt.Errorf("expected non empty sha256 checksum in the response") + } + } + } + + return nil + }) +} + func PutObject_racey_success(s *S3Conf) error { testName := "PutObject_racey_success" runF(testName) @@ -3208,6 +3420,111 @@ func HeadObject_with_contenttype(s *S3Conf) error { }) } +func HeadObject_not_enabled_checksum_mode(s *S3Conf) error { + testName := "HeadObject_not_enabled_checksum_mode" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + _, err := putObjectWithData(500, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32) + } + if res.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C) + } + if res.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1) + } + if res.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256) + } + + return nil + }) +} + +func HeadObject_checksums(s *S3Conf) error { + testName := "HeadObject_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs := []struct { + key string + checksumAlgo types.ChecksumAlgorithm + }{ + { + key: "obj-1", + checksumAlgo: types.ChecksumAlgorithmCrc32, + }, + { + key: "obj-2", + checksumAlgo: types.ChecksumAlgorithmCrc32c, + }, + { + key: "obj-3", + checksumAlgo: types.ChecksumAlgorithmSha1, + }, + { + key: "obj-4", + checksumAlgo: types.ChecksumAlgorithmSha256, + }, + } + + for i, el := range objs { + out, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumAlgorithm: el.checksumAlgo, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { + return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32)) + } + if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C)) + } + if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { + return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1)) + } + if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { + return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256)) + } + } + + return nil + }) +} + func HeadObject_invalid_parent_dir(s *S3Conf) error { testName := "HeadObject_invalid_parent_dir" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -3467,6 +3784,74 @@ func GetObjectAttributes_existing_object(s *S3Conf) error { }) } +func GetObjectAttributes_checksums(s *S3Conf) error { + testName := "GetObjectAttributes_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs := []struct { + key string + checksumAlgo types.ChecksumAlgorithm + }{ + { + key: "obj-1", + checksumAlgo: types.ChecksumAlgorithmCrc32, + }, + { + key: "obj-2", + checksumAlgo: types.ChecksumAlgorithmCrc32c, + }, + { + key: "obj-3", + checksumAlgo: types.ChecksumAlgorithmSha1, + }, + { + key: "obj-4", + checksumAlgo: types.ChecksumAlgorithmSha256, + }, + } + + for i, el := range objs { + out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumAlgorithm: el.checksumAlgo, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &el.key, + ObjectAttributes: []types.ObjectAttributes{ + types.ObjectAttributesChecksum, + }, + }) + cancel() + if err != nil { + return err + } + + if res.Checksum == nil { + return fmt.Errorf("expected non-nil checksum in the response") + } + if getString(res.Checksum.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { + return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.Checksum.ChecksumCRC32)) + } + if getString(res.Checksum.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.Checksum.ChecksumCRC32C)) + } + if getString(res.Checksum.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { + return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.Checksum.ChecksumSHA1)) + } + if getString(res.Checksum.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { + return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.Checksum.ChecksumSHA256)) + } + } + return nil + }) +} + func GetObject_non_existing_key(s *S3Conf) error { testName := "GetObject_non_existing_key" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -3625,6 +4010,111 @@ func GetObject_with_meta(s *S3Conf) error { }) } +func GetObject_not_enabled_checksum_mode(s *S3Conf) error { + testName := "GetObject_not_enabled_checksum_mode" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + _, err := putObjectWithData(350, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32) + } + if res.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C) + } + if res.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1) + } + if res.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256) + } + + return nil + }) +} + +func GetObject_checksums(s *S3Conf) error { + testName := "GetObject_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs := []struct { + key string + checksumAlgo types.ChecksumAlgorithm + }{ + { + key: "obj-1", + checksumAlgo: types.ChecksumAlgorithmCrc32, + }, + { + key: "obj-2", + checksumAlgo: types.ChecksumAlgorithmCrc32c, + }, + { + key: "obj-3", + checksumAlgo: types.ChecksumAlgorithmSha1, + }, + { + key: "obj-4", + checksumAlgo: types.ChecksumAlgorithmSha256, + }, + } + + for i, el := range objs { + out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumAlgorithm: el.checksumAlgo, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { + return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32)) + } + if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C)) + } + if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { + return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1)) + } + if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { + return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256)) + } + } + + return nil + }) +} + func GetObject_large_object(s *S3Conf) error { testName := "GetObject_large_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4160,6 +4650,54 @@ func ListObjects_marker_not_from_obj_list(s *S3Conf) error { }) } +func ListObjects_with_checksum(s *S3Conf) error { + testName := "ListObjects_with_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents := []types.Object{} + + checksumAlgos := types.ChecksumAlgorithmCrc32.Values() + checksumAlgos = append(checksumAlgos, "") + + for i, el := range checksumAlgos { + key := fmt.Sprintf("obj-%v", i) + size := int64(i * 30) + out, err := putObjectWithData(size, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: el, + }, s3client) + if err != nil { + return err + } + + contents = append(contents, types.Object{ + Key: &key, + ETag: out.res.ETag, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{ + el, + }, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(res.Contents, contents) { + return fmt.Errorf("expected the objects list to be %v, instead got %v", contents, res.Contents) + } + + return nil + }) +} + func ListObjects_list_all_objs(s *S3Conf) error { testName := "ListObjects_list_all_objs" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4564,6 +5102,54 @@ func ListObjectsV2_list_all_objs(s *S3Conf) error { }) } +func ListObjectsV2_with_checksum(s *S3Conf) error { + testName := "ListObjectsV2_with_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents := []types.Object{} + + checksumAlgos := types.ChecksumAlgorithmCrc32.Values() + checksumAlgos = append(checksumAlgos, "") + + for i, el := range checksumAlgos { + key := fmt.Sprintf("obj-%v", i) + size := int64(i * 100) + out, err := putObjectWithData(size, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: el, + }, s3client) + if err != nil { + return err + } + + contents = append(contents, types.Object{ + Key: &key, + ETag: out.res.ETag, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{ + el, + }, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(res.Contents, contents) { + return fmt.Errorf("expected the objects list to be %v, instead got %v", contents, res.Contents) + } + + return nil + }) +} + func ListObjectsV2_invalid_parent_prefix(s *S3Conf) error { testName := "ListObjectsV2_invalid_parent_prefix" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -5232,6 +5818,231 @@ func CopyObject_non_existing_dir_object(s *S3Conf) error { }) } +func CopyObject_invalid_checksum_algorithm(s *S3Conf) error { + testName := "CopyObject_invalid_checksum_algorithm" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + ChecksumAlgorithm: types.ChecksumAlgorithm("invalid_checksum_algorithm"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)); err != nil { + return err + } + + return nil + }) +} + +func CopyObject_create_checksum_on_copy(s *S3Conf) error { + testName := "CopyObject_create_checksum_on_copy" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj := "source-object" + dstObj := "destination-object" + _, err := putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }) + cancel() + if err != nil { + return err + } + + if getString(res.CopyObjectResult.ChecksumSHA256) == "" { + return fmt.Errorf("expected non nil sha256 checksum") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if getString(out.ChecksumSHA256) != getString(res.CopyObjectResult.ChecksumSHA256) { + return fmt.Errorf("expected the sha256 checksum to be %v, instead got %v", getString(res.CopyObjectResult.ChecksumSHA256), getString(out.ChecksumSHA256)) + } + + return nil + }) +} + +func CopyObject_should_copy_the_existing_checksum(s *S3Conf) error { + testName := "CopyObject_should_copy_the_existing_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj := "source-object" + dstObj := "destination-object" + out, err := putObjectWithData(100, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32c, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if res.CopyObjectResult.ChecksumCRC32C == nil { + return fmt.Errorf("expected non empty crc32c checksum") + } + if getString(res.CopyObjectResult.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.CopyObjectResult.ChecksumCRC32C)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if getString(resp.ChecksumCRC32C) != getString(res.CopyObjectResult.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(res.CopyObjectResult.ChecksumCRC32C), getString(resp.ChecksumCRC32C)) + } + + return nil + }) +} + +func CopyObject_should_replace_the_existing_checksum(s *S3Conf) error { + testName := "CopyObject_should_replace_the_existing_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj := "source-object" + dstObj := "destination-object" + + _, err := putObjectWithData(100, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, // replace crc32 with sha1 + }) + cancel() + if err != nil { + return err + } + + if res.CopyObjectResult.ChecksumSHA1 == nil { + return fmt.Errorf("expected non empty sha1 checksum") + } + if res.CopyObjectResult.ChecksumCRC32 != nil { + return fmt.Errorf("expected empty crc32 checksum, instead got %v", *res.CopyObjectResult.ChecksumCRC32) + } + + return nil + }) +} + +func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error { + testName := "CopyObject_to_itself_by_replacing_the_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + _, err := putObjectWithData(400, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, // replace sh256 with crc32 + MetadataDirective: types.MetadataDirectiveReplace, + }) + cancel() + if err != nil { + return err + } + + if out.CopyObjectResult.ChecksumCRC32 == nil { + return fmt.Errorf("expected non empty crc32 checksum") + } + if out.CopyObjectResult.ChecksumCRC32C != nil { + return fmt.Errorf("expected empty crc32c checksum") + } + if out.CopyObjectResult.ChecksumSHA1 != nil { + return fmt.Errorf("expected empty sha1 checksum") + } + if out.CopyObjectResult.ChecksumSHA256 != nil { + return fmt.Errorf("expected empty sha256 checksum") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumCRC32 == nil { + return fmt.Errorf("expected non empty crc32 checksum") + } + if res.ChecksumCRC32C != nil { + return fmt.Errorf("expected empty crc32c checksum") + } + if res.ChecksumSHA1 != nil { + return fmt.Errorf("expected empty sha1 checksum") + } + if res.ChecksumSHA256 != nil { + return fmt.Errorf("expected empty sha256 checksum") + } + + return nil + }) +} + func CopyObject_success(s *S3Conf) error { testName := "CopyObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -5875,6 +6686,46 @@ func CreateMultipartUpload_past_retain_until_date(s *S3Conf) error { }) } +func CreateMultipartUpload_invalid_checksum_algorithm(s *S3Conf) error { + testName := "CreateMultipartUpload_invalid_checksum_algorithm" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + ChecksumAlgorithm: types.ChecksumAlgorithm("invalid_checksum_algorithm"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)); err != nil { + return err + } + + return nil + }) +} + +func CreateMultipartUpload_valid_checksum_algorithm(s *S3Conf) error { + testName := "CreateMultipartUpload_valid_checksum_algorithm" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32c, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumAlgorithm != types.ChecksumAlgorithmCrc32c { + return fmt.Errorf("expected the checksum algorithm to be %v, instead got %v", types.ChecksumAlgorithmCrc32c, res.ChecksumAlgorithm) + } + + return nil + }) +} + func CreateMultipartUpload_with_invalid_tagging(s *S3Conf) error { testName := "CreateMultipartUpload_with_invalid_tagging" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -6045,6 +6896,366 @@ func UploadPart_non_existing_mp_upload(s *S3Conf) error { }) } +func UploadPart_checksum_algorithm_and_header_mismatch(s *S3Conf) error { + testName := "UploadPart_checksum_algorithm_and_header_mismatch" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + ChecksumCRC32C: getPtr("m0cB1Q=="), + PartNumber: &partNumber, + UploadId: mp.UploadId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_multiple_checksum_headers(s *S3Conf) error { + testName := "UploadPart_multiple_checksum_headers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32c)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumSHA1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), + ChecksumCRC32C: getPtr("m0cB1Q=="), + UploadId: mp.UploadId, + PartNumber: &partNumber, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_invalid_checksum_header(s *S3Conf) error { + testName := "UploadPart_invalid_checksum_header" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha1)) + if err != nil { + return err + } + + partNumber := int32(1) + + for _, el := range []struct { + algo string + crc32 *string + crc32c *string + sha1 *string + sha256 *string + }{ + // CRC32 tests + { + algo: "crc32", + crc32: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "crc32", + crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32 + }, + // CRC32C tests + { + algo: "crc32c", + crc32c: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "crc32c", + crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c + }, + // SHA1 tests + { + algo: "sha1", + sha1: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "sha1", + sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1 + }, + // SHA256 tests + { + algo: "sha256", + sha256: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "sha256", + sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56 + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + PartNumber: &partNumber, + UploadId: mp.UploadId, + }) + cancel() + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", el.algo))); err != nil { + return err + } + } + + return nil + }) +} + +func UploadPart_checksum_algorithm_mistmatch_on_initialization(s *S3Conf) error { + testName := "UploadPart_checksum_algorithm_mistmatch_on_initialization" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr(types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmSha1)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s *S3Conf) error { + testName := "UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + ChecksumSHA256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="), + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr(types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmSha256)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_required_checksum(s *S3Conf) error { + testName := "UploadPart_required_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32c)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr(types.ChecksumAlgorithmCrc32c, "null")); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_null_checksum(s *S3Conf) error { + testName := "UploadPart_null_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr("null", types.ChecksumAlgorithmCrc32)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_incorrect_checksums(s *S3Conf) error { + testName := "UploadPart_incorrect_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for _, el := range []struct { + algo types.ChecksumAlgorithm + crc32 *string + crc32c *string + sha1 *string + sha256 *string + }{ + { + algo: types.ChecksumAlgorithmCrc32, + crc32: getPtr("DUoRhQ=="), + }, + { + algo: types.ChecksumAlgorithmCrc32c, + crc32c: getPtr("yZRlqg=="), + }, + { + algo: types.ChecksumAlgorithmSha1, + sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), + }, + { + algo: types.ChecksumAlgorithmSha256, + sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="), + }, + } { + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo)) + if err != nil { + return err + } + + body := strings.NewReader("random string body") + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + UploadId: mp.UploadId, + PartNumber: &partNumber, + Body: body, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + return err + } + } + + return nil + }) +} + +func UploadPart_with_checksums_success(s *S3Conf) error { + testName := "UploadPart_with_checksums_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) + if err != nil { + return err + } + + partNumber := int32(1) + data := make([]byte, i*100) + rand.Read(data) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: algo, + UploadId: mp.UploadId, + PartNumber: &partNumber, + Body: bytes.NewReader(data), + }) + cancel() + if err != nil { + return err + } + + switch algo { + case types.ChecksumAlgorithmCrc32: + if res.ChecksumCRC32 == nil { + return fmt.Errorf("expected non empty crc32 checksum in the response") + } + case types.ChecksumAlgorithmCrc32c: + if res.ChecksumCRC32C == nil { + return fmt.Errorf("expected non empty crc32c checksum in the response") + } + case types.ChecksumAlgorithmSha1: + if res.ChecksumSHA1 == nil { + return fmt.Errorf("expected non empty sha1 checksum in the response") + } + case types.ChecksumAlgorithmSha256: + if res.ChecksumSHA256 == nil { + return fmt.Errorf("expected non empty sha256 checksum in the response") + } + } + } + + return nil + }) +} + func UploadPart_non_existing_key(s *S3Conf) error { testName := "UploadPart_non_existing_key" partNumber := int32(1) @@ -6547,6 +7758,162 @@ func UploadPartCopy_by_range_success(s *S3Conf) error { }) } +func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error { + testName := "UploadPartCopy_should_copy_the_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + srcObj := "source-object" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + out, err := putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + }, s3client) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if getString(res.CopyPartResult.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { + return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.CopyPartResult.ChecksumCRC32)) + } + if res.CopyPartResult.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32C) + } + if res.CopyPartResult.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA1) + } + if res.CopyPartResult.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA256) + } + + return nil + }) +} + +func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error { + testName := "UploadPartCopy_should_not_copy_the_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + srcObj := "source-object" + + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + _, err = putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }, s3client) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if res.CopyPartResult.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32) + } + if res.CopyPartResult.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32C) + } + if res.CopyPartResult.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA1) + } + if res.CopyPartResult.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA256) + } + + return nil + }) +} + +func UploadPartCopy_should_calculate_the_checksum(s *S3Conf) error { + testName := "UploadPartCopy_should_calculate_the_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + srcObj := "source-object" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha256)) + if err != nil { + return err + } + + _, err = putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, // different from the mp checksum (sha256) + }, s3client) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if res.CopyPartResult.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32) + } + if res.CopyPartResult.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32C) + } + if res.CopyPartResult.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA1) + } + if getString(res.CopyPartResult.ChecksumSHA256) == "" { + return fmt.Errorf("expected non empty sha256 checksum") + } + + return nil + }) +} + func ListParts_incorrect_uploadId(s *S3Conf) error { testName := "ListParts_incorrect_uploadId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -6707,6 +8074,42 @@ func ListParts_truncated(s *S3Conf) error { }) } +func ListParts_with_checksums(s *S3Conf) error { + testName := "ListParts_with_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, int64((i+1)*5*1024*1024), int64(i+1), bucket, obj, *mp.UploadId, withChecksum(algo)) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + }) + cancel() + if err != nil { + return err + } + + if !compareParts(parts, res.Parts) { + return fmt.Errorf("expected the mp parts to be %v, instead got %v", parts, res.Parts) + } + } + + return nil + }) +} + func ListParts_success(s *S3Conf) error { testName := "ListParts_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -6914,6 +8317,65 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error { }) } +func ListMultipartUploads_with_checksums(s *S3Conf) error { + testName := "ListMultipartUploads_with_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + uploads := []types.MultipartUpload{} + for _, el := range []struct { + obj string + algo types.ChecksumAlgorithm + }{ + { + obj: "obj-1", + algo: types.ChecksumAlgorithmCrc32, + }, + { + obj: "obj-2", + algo: types.ChecksumAlgorithmCrc32c, + }, + { + obj: "obj-3", + algo: types.ChecksumAlgorithmSha1, + }, + { + obj: "obj-4", + algo: types.ChecksumAlgorithmSha256, + }, + { + obj: "obj-5", + }, + } { + key := el.obj + mp, err := createMp(s3client, bucket, key, withChecksum(el.algo)) + if err != nil { + return err + } + + uploads = append(uploads, types.MultipartUpload{ + Key: &key, + UploadId: mp.UploadId, + StorageClass: types.StorageClassStandard, + ChecksumAlgorithm: el.algo, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareMultipartUploads(res.Uploads, uploads) { + return fmt.Errorf("expected the final multipart uploads to be %v, instead got %v", uploads, res.Uploads) + } + + return nil + }) +} + func ListMultipartUploads_success(s *S3Conf) error { testName := "ListMultipartUploads_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/utils.go b/tests/integration/utils.go index c02f30c..5452f6a 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -18,11 +18,15 @@ import ( "bytes" "context" "crypto/rand" + "crypto/sha1" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/xml" "errors" "fmt" + "hash" + "hash/crc32" "io" "math/big" rnd "math/rand" @@ -475,11 +479,26 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) }, nil } -func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) { +type mpCfg struct { + checksumAlgorithm types.ChecksumAlgorithm +} + +type mpOpt func(*mpCfg) + +func withChecksum(algo types.ChecksumAlgorithm) mpOpt { + return func(mc *mpCfg) { mc.checksumAlgorithm = algo } +} + +func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) { + cfg := new(mpCfg) + for _, opt := range opts { + opt(cfg) + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &key, + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: cfg.checksumAlgorithm, }) cancel() return out, err @@ -513,6 +532,9 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { if item.StorageClass != list2[i].StorageClass { return false } + if item.ChecksumAlgorithm != list2[i].ChecksumAlgorithm { + return false + } } return true @@ -530,6 +552,21 @@ func compareParts(parts1, parts2 []types.Part) bool { if *prt.ETag != *parts2[i].ETag { return false } + if *prt.Size != *parts2[i].Size { + return false + } + if getString(prt.ChecksumCRC32) != getString(parts2[i].ChecksumCRC32) { + return false + } + if getString(prt.ChecksumCRC32C) != getString(parts2[i].ChecksumCRC32C) { + return false + } + if getString(prt.ChecksumSHA1) != getString(parts2[i].ChecksumSHA1) { + return false + } + if getString(prt.ChecksumSHA256) != getString(parts2[i].ChecksumSHA256) { + return false + } } return true } @@ -647,6 +684,13 @@ func compareObjects(list1, list2 []types.Object) bool { *obj.Key, *list2[i].Key, obj.StorageClass, list2[i].StorageClass) return false } + if len(obj.ChecksumAlgorithm) != 0 { + if obj.ChecksumAlgorithm[0] != list2[i].ChecksumAlgorithm[0] { + fmt.Printf("checksum algorithms are not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, obj.ChecksumAlgorithm[0], list2[i].ChecksumAlgorithm[0]) + return false + } + } } return true @@ -711,10 +755,28 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool { return true } -func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string) (parts []types.Part, csum string, err error) { +func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) { partSize := size / partCount - hash := sha256.New() + var hash hash.Hash + + cfg := new(mpCfg) + for _, opt := range opts { + opt(cfg) + } + + switch cfg.checksumAlgorithm { + case types.ChecksumAlgorithmCrc32: + hash = crc32.NewIEEE() + case types.ChecksumAlgorithmCrc32c: + hash = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case types.ChecksumAlgorithmSha1: + hash = sha1.New() + case types.ChecksumAlgorithmSha256: + hash = sha256.New() + default: + hash = sha256.New() + } for partNumber := int64(1); partNumber <= partCount; partNumber++ { partStart := (partNumber - 1) * partSize @@ -730,11 +792,12 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) pn := int32(partNumber) out, err := client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &key, - UploadId: &uploadId, - Body: bytes.NewReader(partBuffer), - PartNumber: &pn, + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + Body: bytes.NewReader(partBuffer), + PartNumber: &pn, + ChecksumAlgorithm: cfg.checksumAlgorithm, }) cancel() if err != nil { @@ -742,14 +805,22 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId } parts = append(parts, types.Part{ - ETag: out.ETag, - PartNumber: &pn, - Size: &partSize, + ETag: out.ETag, + PartNumber: &pn, + Size: &partSize, + ChecksumCRC32: out.ChecksumCRC32, + ChecksumCRC32C: out.ChecksumCRC32C, + ChecksumSHA1: out.ChecksumSHA1, + ChecksumSHA256: out.ChecksumSHA256, }) } - sum := hash.Sum(nil) - csum = hex.EncodeToString(sum[:]) + + if cfg.checksumAlgorithm == "" { + csum = hex.EncodeToString(sum[:]) + } else { + csum = base64.StdEncoding.EncodeToString(sum[:]) + } return parts, csum, err }