diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 27a433d3..102e94cd 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -905,9 +905,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult } // Each part is translated into an uncommitted block in a newly created blob in staging area -func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) { +func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil { - return "", err + return nil, err } // TODO: request streamable version of StageBlock() @@ -916,32 +916,34 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (eta // the body in memory to create an io.ReadSeekCloser rdr, err := getReadSeekCloser(input.Body) if err != nil { - return "", err + return nil, err } client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) if err != nil { - return "", err + return nil, err } // block id serves as etag here - etag = blockIDInt32ToBase64(*input.PartNumber) + etag := blockIDInt32ToBase64(*input.PartNumber) _, err = client.StageBlock(ctx, etag, rdr, nil) if err != nil { - return "", parseMpError(err) + return nil, parseMpError(err) } - return etag, nil + return &s3.UploadPartOutput{ + ETag: &etag, + }, nil } -func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) if err != nil { - return s3response.CopyObjectResult{}, nil + return s3response.CopyPartResult{}, nil } if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil { - return s3response.CopyObjectResult{}, err + return s3response.CopyPartResult{}, err } eTag := blockIDInt32ToBase64(*input.PartNumber) @@ -949,10 +951,10 @@ func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInp //TODO: the action returns not implemented on azurite, maybe in production this will work? _, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil) if err != nil { - return s3response.CopyObjectResult{}, parseMpError(err) + return s3response.CopyPartResult{}, parseMpError(err) } - return s3response.CopyObjectResult{}, nil + return s3response.CopyPartResult{}, nil } // Lists all uncommitted parts from the blob @@ -1195,26 +1197,46 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } + uncommittedBlocks := map[int32]*blockblob.Block{} + for _, el := range blockList.UncommittedBlocks { + ptNumber, err := decodeBlockId(backend.GetStringFromPtr(el.Name)) + if err != nil { + return nil, fmt.Errorf("invalid block name: %w", err) + } + + uncommittedBlocks[int32(ptNumber)] = el + } + slices.SortFunc(blockList.UncommittedBlocks, func(a *blockblob.Block, b *blockblob.Block) int { ptNumber, _ := decodeBlockId(*a.Name) nextPtNumber, _ := decodeBlockId(*b.Name) return ptNumber - nextPtNumber }) + // The initialie values is the lower limit of partNumber: 0 + var partNumber int32 last := len(blockList.UncommittedBlocks) - 1 - for i, block := range blockList.UncommittedBlocks { - ptNumber, err := decodeBlockId(*block.Name) - if err != nil { + for i, part := range input.MultipartUpload.Parts { + if part.PartNumber == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidPart) + } + if *part.PartNumber < 1 { + return nil, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber) + } + if *part.PartNumber <= partNumber { + return nil, s3err.GetAPIError(s3err.ErrInvalidPartOrder) + } + partNumber = *part.PartNumber + + block, ok := uncommittedBlocks[*part.PartNumber] + if !ok { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } - if *input.MultipartUpload.Parts[i].ETag != *block.Name { + if *part.ETag != *block.Name { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } - if *input.MultipartUpload.Parts[i].PartNumber != int32(ptNumber) { - return nil, s3err.GetAPIError(s3err.ErrInvalidPart) - } - // all parts except the last need to be greater, thena + // all parts except the last need to be greater, than // the minimum allowed size (5 Mib) if i < last && *block.Size < backend.MinPartSize { return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall) diff --git a/backend/backend.go b/backend/backend.go index 1141d648..2f4e67a4 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -53,8 +53,8 @@ type Backend interface { AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) - UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) - UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) + UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) + UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) // standard object operations PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) @@ -166,11 +166,11 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) { return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) { - return "", s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) +func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { diff --git a/backend/posix/posix.go b/backend/posix/posix.go index ee1c1675..6e4d2d42 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "io/fs" + "net/http" "os" "path/filepath" "sort" @@ -39,6 +40,7 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/backend/meta" + "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -87,6 +89,7 @@ const ( aclkey = "acl" ownershipkey = "ownership" etagkey = "etag" + checksumsKey = "checksums" policykey = "policy" bucketLockKey = "bucket-lock" objectRetentionKey = "object-retention" @@ -1308,6 +1311,20 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa } } + // Set object checksum algorithm + if mpu.ChecksumAlgorithm != "" { + err := p.storeChecksums(nil, bucket, filepath.Join(objdir, uploadID), s3response.Checksum{ + Algorithm: mpu.ChecksumAlgorithm, + Type: mpu.ChecksumType, + }) + if err != nil { + // cleanup object if returning error + _ = os.RemoveAll(filepath.Join(tmppath, uploadID)) + _ = os.Remove(tmppath) + return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("store mp checksum algorithm: %w", err) + } + } + return s3response.InitiateMultipartUploadResult{ Bucket: bucket, Key: object, @@ -1334,6 +1351,21 @@ func (p *Posix) getChownIDs(acct auth.Account) (int, int, bool) { return uid, gid, needsChown } +func getPartChecksum(algo types.ChecksumAlgorithm, part types.CompletedPart) string { + switch algo { + case types.ChecksumAlgorithmCrc32: + return backend.GetStringFromPtr(part.ChecksumCRC32) + case types.ChecksumAlgorithmCrc32c: + return backend.GetStringFromPtr(part.ChecksumCRC32C) + case types.ChecksumAlgorithmSha1: + return backend.GetStringFromPtr(part.ChecksumSHA1) + case types.ChecksumAlgorithmSha256: + return backend.GetStringFromPtr(part.ChecksumSHA256) + default: + return "" + } +} + func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { @@ -1373,13 +1405,43 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get mp checksums: %w", err) + } + var checksumAlgorithm types.ChecksumAlgorithm + if checksums.Algorithm != "" { + checksumAlgorithm = checksums.Algorithm + } + + // ChecksumType should be the same as specified on CreateMultipartUpload + if checksums.Type != input.ChecksumType { + checksumType := checksums.Type + if checksumType == "" { + checksumType = types.ChecksumType("null") + } + + return nil, s3err.GetChecksumTypeMismatchOnMpErr(checksumType) + } + // check all parts ok last := len(parts) - 1 var totalsize int64 + + // The initialie values is the lower limit of partNumber: 0 + var partNumber int32 for i, part := range parts { - if part.PartNumber == nil || *part.PartNumber < 1 { + if part.PartNumber == nil { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } + if *part.PartNumber < 1 { + return nil, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber) + } + if *part.PartNumber <= partNumber { + return nil, s3err.GetAPIError(s3err.ErrInvalidPartOrder) + } + + partNumber = *part.PartNumber partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber)) fullPartPath := filepath.Join(bucket, partObjPath) @@ -1403,6 +1465,32 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM if parts[i].ETag == nil || etag != *parts[i].ETag { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } + + partChecksum, err := p.retrieveChecksums(nil, bucket, partObjPath) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get part checksum: %w", err) + } + + // If checksum has been provided on mp initalization + err = validatePartChecksum(partChecksum, part) + if err != nil { + return nil, err + } + } + + var hashRdr *utils.HashReader + var compositeChecksumRdr *utils.CompositeChecksumReader + switch checksums.Type { + case types.ChecksumTypeFullObject: + hashRdr, err = utils.NewHashReader(nil, "", utils.HashType(strings.ToLower(string(checksumAlgorithm)))) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + case types.ChecksumTypeComposite: + compositeChecksumRdr, err = utils.NewCompositeChecksumReader(utils.HashType(strings.ToLower(string(checksumAlgorithm)))) + if err != nil { + return nil, fmt.Errorf("initialize composite checksum reader: %w", err) + } } f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, @@ -1416,17 +1504,25 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM defer f.cleanup() for _, part := range parts { - if part.PartNumber == nil || *part.PartNumber < 1 { - return nil, s3err.GetAPIError(s3err.ErrInvalidPart) - } - partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber)) fullPartPath := filepath.Join(bucket, partObjPath) pf, err := os.Open(fullPartPath) if err != nil { return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err) } - _, err = io.Copy(f.File(), pf) + + var rdr io.Reader = pf + if checksums.Type == types.ChecksumTypeFullObject { + hashRdr.SetReader(rdr) + rdr = hashRdr + } else if checksums.Type == types.ChecksumTypeComposite { + err := compositeChecksumRdr.Process(getPartChecksum(checksumAlgorithm, part)) + if err != nil { + return nil, fmt.Errorf("process %v part checksum: %w", *part.PartNumber, err) + } + } + + _, err = io.Copy(f.File(), rdr) pf.Close() if err != nil { if errors.Is(err, syscall.EDQUOT) { @@ -1524,6 +1620,65 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } } + var crc32 *string + var crc32c *string + var sha1 *string + var sha256 *string + var crc64nvme *string + + // Calculate, compare with the provided checksum and store them + if checksums.Type != "" { + checksum := s3response.Checksum{ + Algorithm: checksumAlgorithm, + Type: checksums.Type, + } + + var sum string + switch checksums.Type { + case types.ChecksumTypeComposite: + sum = compositeChecksumRdr.Sum() + case types.ChecksumTypeFullObject: + sum = hashRdr.Sum() + } + + switch checksumAlgorithm { + case types.ChecksumAlgorithmCrc32: + if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 != sum { + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + checksum.CRC32 = &sum + crc32 = &sum + case types.ChecksumAlgorithmCrc32c: + if input.ChecksumCRC32C != nil && *input.ChecksumCRC32C != sum { + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + checksum.CRC32C = &sum + crc32c = &sum + case types.ChecksumAlgorithmSha1: + if input.ChecksumSHA1 != nil && *input.ChecksumSHA1 != sum { + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + checksum.SHA1 = &sum + sha1 = &sum + case types.ChecksumAlgorithmSha256: + if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 != sum { + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + checksum.SHA256 = &sum + sha256 = &sum + case types.ChecksumAlgorithmCrc64nvme: + if input.ChecksumCRC64NVME != nil && *input.ChecksumCRC64NVME != sum { + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + checksum.CRC64NVME = &sum + crc64nvme = &sum + } + err := p.storeChecksums(f.File(), bucket, object, checksum) + if err != nil { + return nil, fmt.Errorf("store object checksum: %w", err) + } + } + // load and set retention ret, err := p.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -1556,13 +1711,97 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM os.Remove(filepath.Join(bucket, objdir)) return &s3.CompleteMultipartUploadOutput{ - Bucket: &bucket, - ETag: &s3MD5, - Key: &object, - VersionId: &versionID, + Bucket: &bucket, + ETag: &s3MD5, + Key: &object, + VersionId: &versionID, + ChecksumCRC32: crc32, + ChecksumCRC32C: crc32c, + ChecksumSHA1: sha1, + ChecksumSHA256: sha256, + ChecksumCRC64NVME: crc64nvme, + ChecksumType: checksums.Type, }, nil } +func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart) error { + n := numberOfChecksums(part) + if n > 1 { + return s3err.GetAPIError(s3err.ErrInvalidChecksumPart) + } + if checksum.Algorithm == "" { + if n != 0 { + return s3err.GetAPIError(s3err.ErrInvalidPart) + } + + return nil + } + + algo := checksum.Algorithm + if n == 0 { + return s3err.APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The upload was created using a %v checksum. The complete request must include the checksum for each part. It was missing for part %v in the request.", strings.ToLower(string(algo)), *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + + for _, cs := range []struct { + checksum *string + expectedChecksum string + algo types.ChecksumAlgorithm + }{ + {part.ChecksumCRC32, getString(checksum.CRC32), types.ChecksumAlgorithmCrc32}, + {part.ChecksumCRC32C, getString(checksum.CRC32C), types.ChecksumAlgorithmCrc32c}, + {part.ChecksumSHA1, getString(checksum.SHA1), types.ChecksumAlgorithmSha1}, + {part.ChecksumSHA256, getString(checksum.SHA256), types.ChecksumAlgorithmSha256}, + {part.ChecksumCRC64NVME, getString(checksum.CRC64NVME), types.ChecksumAlgorithmCrc64nvme}, + } { + if cs.checksum == nil { + continue + } + + if !utils.IsValidChecksum(*cs.checksum, cs.algo) { + return s3err.GetAPIError(s3err.ErrInvalidChecksumPart) + } + + if *cs.checksum != cs.expectedChecksum { + if algo == cs.algo { + return s3err.GetAPIError(s3err.ErrInvalidPart) + } + + return s3err.APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The %v you specified for part %v did not match what we received.", strings.ToLower(string(cs.algo)), *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + + return nil +} + +func numberOfChecksums(part types.CompletedPart) int { + counter := 0 + if getString(part.ChecksumCRC32) != "" { + counter++ + } + if getString(part.ChecksumCRC32C) != "" { + counter++ + } + if getString(part.ChecksumSHA1) != "" { + counter++ + } + if getString(part.ChecksumSHA256) != "" { + counter++ + } + if getString(part.ChecksumCRC64NVME) != "" { + counter++ + } + + return counter +} + func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) { sum := sha256.Sum256([]byte(object)) objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) @@ -1752,11 +1991,19 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl if keyMarkerInd == -1 && objectName == keyMarker { keyMarkerInd = len(uploads) } + + checksum, err := p.retrieveChecksums(nil, bucket, filepath.Join(metaTmpMultipartDir, obj.Name(), uploadID)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return lmu, fmt.Errorf("get mp checksum: %w", err) + } + uploads = append(uploads, s3response.Upload{ - Key: objectName, - UploadID: uploadID, - StorageClass: types.StorageClassStandard, - Initiated: fi.ModTime(), + Key: objectName, + UploadID: uploadID, + StorageClass: types.StorageClassStandard, + Initiated: fi.ModTime(), + ChecksumAlgorithm: checksum.Algorithm, + ChecksumType: checksum.Type, }) } } @@ -1875,6 +2122,11 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon return lpr, fmt.Errorf("readdir upload: %w", err) } + checksum, err := p.retrieveChecksums(nil, tmpdir, uploadID) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return lpr, fmt.Errorf("get mp checksum: %w", err) + } + var parts []s3response.Part for _, e := range ents { pn, err := strconv.Atoi(e.Name()) @@ -1893,16 +2145,26 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon etag = "" } + checksum, err := p.retrieveChecksums(nil, bucket, partPath) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + continue + } + fi, err := os.Lstat(filepath.Join(bucket, partPath)) if err != nil { continue } parts = append(parts, s3response.Part{ - PartNumber: pn, - ETag: etag, - LastModified: fi.ModTime(), - Size: fi.Size(), + PartNumber: pn, + ETag: etag, + LastModified: fi.ModTime(), + Size: fi.Size(), + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, + ChecksumCRC64NVME: checksum.CRC64NVME, }) } @@ -1934,20 +2196,27 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon Parts: parts, UploadID: uploadID, StorageClass: types.StorageClassStandard, + ChecksumAlgorithm: checksum.Algorithm, + ChecksumType: checksum.Type, }, nil } -func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (string, error) { +type hashConfig struct { + value *string + hashType utils.HashType +} + +func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} } if input.Bucket == nil { - return "", s3err.GetAPIError(s3err.ErrInvalidBucketName) + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) } if input.Key == nil { - return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } bucket := *input.Bucket @@ -1962,79 +2231,171 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (stri _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { - return "", s3err.GetAPIError(s3err.ErrNoSuchBucket) + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return "", fmt.Errorf("stat bucket: %w", err) + return nil, fmt.Errorf("stat bucket: %w", err) } sum := sha256.Sum256([]byte(object)) objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + mpPath := filepath.Join(objdir, uploadID) - _, err = os.Stat(filepath.Join(bucket, objdir, uploadID)) + _, err = os.Stat(filepath.Join(bucket, mpPath)) if errors.Is(err, fs.ErrNotExist) { - return "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) } if err != nil { - return "", fmt.Errorf("stat uploadid: %w", err) + return nil, fmt.Errorf("stat uploadid: %w", err) } - partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part)) + partPath := filepath.Join(mpPath, fmt.Sprintf("%v", *part)) f, err := p.openTmpFile(filepath.Join(bucket, objdir), bucket, partPath, length, acct, doFalloc) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", fmt.Errorf("open temp file: %w", err) + return nil, fmt.Errorf("open temp file: %w", err) } defer f.cleanup() hash := md5.New() tr := io.TeeReader(r, hash) + + hashConfigs := []hashConfig{ + {input.ChecksumCRC32, utils.HashTypeCRC32}, + {input.ChecksumCRC32C, utils.HashTypeCRC32C}, + {input.ChecksumSHA1, utils.HashTypeSha1}, + {input.ChecksumSHA256, utils.HashTypeSha256}, + {input.ChecksumCRC64NVME, utils.HashTypeCRC64NVME}, + } + + var hashRdr *utils.HashReader + for _, config := range hashConfigs { + if config.value != nil { + hashRdr, err = utils.NewHashReader(tr, *config.value, config.hashType) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + } + + // If only the checksum algorithm is provided register + // a new HashReader to calculate the object checksum + if hashRdr == nil && input.ChecksumAlgorithm != "" { + hashRdr, err = utils.NewHashReader(tr, "", utils.HashType(strings.ToLower(string(input.ChecksumAlgorithm)))) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + + checksums, chErr := p.retrieveChecksums(nil, bucket, mpPath) + if chErr != nil && !errors.Is(chErr, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("retreive mp checksum: %w", chErr) + } + + // If checksum isn't provided for the part, + // but it has been provided on mp initalization + if hashRdr == nil && chErr == nil && checksums.Algorithm != "" { + return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithm, "null") + } + + // Check if the provided checksum algorithm match + // the one specified on mp initialization + if hashRdr != nil && chErr == nil && checksums.Type != "" { + algo := types.ChecksumAlgorithm(strings.ToUpper(string(hashRdr.Type()))) + if checksums.Algorithm != algo { + return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithm, algo) + } + } + _, err = io.Copy(f, tr) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return "", fmt.Errorf("write part data: %w", err) + return nil, fmt.Errorf("write part data: %w", err) } dataSum := hash.Sum(nil) etag := hex.EncodeToString(dataSum) err = p.meta.StoreAttribute(f.File(), bucket, partPath, etagkey, []byte(etag)) if err != nil { - return "", fmt.Errorf("set etag attr: %w", err) + return nil, fmt.Errorf("set etag attr: %w", err) + } + + res := &s3.UploadPartOutput{ + ETag: &etag, + } + + if hashRdr != nil { + checksum := s3response.Checksum{ + Algorithm: input.ChecksumAlgorithm, + } + + // Validate the provided checksum + sum := hashRdr.Sum() + switch hashRdr.Type() { + case utils.HashTypeCRC32: + checksum.CRC32 = &sum + res.ChecksumCRC32 = &sum + case utils.HashTypeCRC32C: + checksum.CRC32C = &sum + res.ChecksumCRC32C = &sum + case utils.HashTypeSha1: + checksum.SHA1 = &sum + res.ChecksumSHA1 = &sum + case utils.HashTypeSha256: + checksum.SHA256 = &sum + res.ChecksumSHA256 = &sum + case utils.HashTypeCRC64NVME: + checksum.CRC64NVME = &sum + res.ChecksumCRC64NVME = &sum + } + + // Store the checksums if the checksum type has been + // specified on mp initialization + if checksums.Type != "" { + err := p.storeChecksums(f.File(), bucket, partPath, checksum) + if err != nil { + return nil, fmt.Errorf("store checksum: %w", err) + } + } } err = f.link() if err != nil { - return "", fmt.Errorf("link object in namespace: %w", err) + return nil, fmt.Errorf("link object in namespace: %w", err) } - return etag, nil + return res, nil } -func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} } if upi.Bucket == nil { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } if upi.Key == nil { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } _, err := os.Stat(*upi.Bucket) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat bucket: %w", err) } sum := sha256.Sum256([]byte(*upi.Key)) @@ -2042,46 +2403,46 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) _, err = os.Stat(filepath.Join(*upi.Bucket, objdir, *upi.UploadId)) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchUpload) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchUpload) } if errors.Is(err, syscall.ENAMETOOLONG) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrKeyTooLong) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrKeyTooLong) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat uploadid: %w", err) } partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber)) srcBucket, srcObject, srcVersionId, err := backend.ParseCopySource(*upi.CopySource) if err != nil { - return s3response.CopyObjectResult{}, err + return s3response.CopyPartResult{}, err } _, err = os.Stat(srcBucket) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat bucket: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat bucket: %w", err) } vStatus, err := p.getBucketVersioningStatus(ctx, srcBucket) if err != nil { - return s3response.CopyObjectResult{}, err + return s3response.CopyPartResult{}, err } vEnabled := p.isBucketVersioningEnabled(vStatus) if srcVersionId != "" { if !p.versioningEnabled() || !vEnabled { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidVersionId) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrInvalidVersionId) } vId, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, versionIdKey) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return s3response.CopyObjectResult{}, fmt.Errorf("get src object version id: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("get src object version id: %w", err) } if string(vId) != srcVersionId { @@ -2094,20 +2455,20 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) fi, err := os.Stat(objPath) if errors.Is(err, fs.ErrNotExist) { if p.versioningEnabled() && vEnabled { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchVersion) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchVersion) } - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if errors.Is(err, syscall.ENAMETOOLONG) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrKeyTooLong) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrKeyTooLong) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat object: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat object: %w", err) } startOffset, length, err := backend.ParseRange(fi.Size(), *upi.CopySourceRange) if err != nil { - return s3response.CopyObjectResult{}, err + return s3response.CopyPartResult{}, err } if length == -1 { @@ -2115,25 +2476,25 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) } if startOffset+length > fi.Size()+1 { - return s3response.CopyObjectResult{}, backend.CreateExceedingRangeErr(fi.Size()) + return s3response.CopyPartResult{}, backend.CreateExceedingRangeErr(fi.Size()) } f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir), *upi.Bucket, partPath, length, acct, doFalloc) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } - return s3response.CopyObjectResult{}, fmt.Errorf("open temp file: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("open temp file: %w", err) } defer f.cleanup() srcf, err := os.Open(objPath) if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("open object: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err) } defer srcf.Close() @@ -2141,35 +2502,101 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) hash := md5.New() tr := io.TeeReader(rdr, hash) + mpChecksums, err := p.retrieveChecksums(nil, *upi.Bucket, filepath.Join(objdir, *upi.UploadId)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyPartResult{}, fmt.Errorf("retreive mp checksums: %w", err) + } + + checksums, err := p.retrieveChecksums(nil, objPath, "") + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyPartResult{}, fmt.Errorf("retreive object part checksums: %w", err) + } + + // TODO: Should the checksum be recalculated or just copied ? + var hashRdr *utils.HashReader + if mpChecksums.Algorithm != "" { + if checksums.Algorithm == "" || mpChecksums.Algorithm != checksums.Algorithm { + hashRdr, err = utils.NewHashReader(tr, "", utils.HashType(strings.ToLower(string(mpChecksums.Algorithm)))) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + } + _, err = io.Copy(f, tr) if err != nil { if errors.Is(err, syscall.EDQUOT) { - return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + return s3response.CopyPartResult{}, fmt.Errorf("copy part data: %w", err) + } + + if checksums.Algorithm != "" { + if mpChecksums.Algorithm == "" { + checksums = s3response.Checksum{} + } else { + if hashRdr == nil { + err := p.storeChecksums(f.File(), objPath, "", checksums) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("store part checksum: %w", err) + } + } + } + } + if hashRdr != nil { + algo := types.ChecksumAlgorithm(strings.ToUpper(string(hashRdr.Type()))) + checksums = s3response.Checksum{ + Algorithm: algo, + } + + sum := hashRdr.Sum() + switch algo { + case types.ChecksumAlgorithmCrc32: + checksums.CRC32 = &sum + case types.ChecksumAlgorithmCrc32c: + checksums.CRC32C = &sum + case types.ChecksumAlgorithmSha1: + checksums.SHA1 = &sum + case types.ChecksumAlgorithmSha256: + checksums.SHA256 = &sum + case types.ChecksumAlgorithmCrc64nvme: + checksums.CRC64NVME = &sum + } + + err := p.storeChecksums(f.File(), objPath, "", checksums) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("store part checksum: %w", err) } - return s3response.CopyObjectResult{}, fmt.Errorf("copy part data: %w", err) } dataSum := hash.Sum(nil) etag := hex.EncodeToString(dataSum) err = p.meta.StoreAttribute(f.File(), *upi.Bucket, partPath, etagkey, []byte(etag)) if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("set etag attr: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("set etag attr: %w", err) } err = f.link() if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("link object in namespace: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("link object in namespace: %w", err) } fi, err = os.Stat(filepath.Join(*upi.Bucket, partPath)) if err != nil { - return s3response.CopyObjectResult{}, fmt.Errorf("stat part path: %w", err) + return s3response.CopyPartResult{}, fmt.Errorf("stat part path: %w", err) } - return s3response.CopyObjectResult{ - ETag: etag, + return s3response.CopyPartResult{ + ETag: &etag, LastModified: fi.ModTime(), CopySourceVersionId: srcVersionId, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, }, nil } @@ -2307,6 +2734,38 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons hash := md5.New() rdr := io.TeeReader(po.Body, hash) + + hashConfigs := []hashConfig{ + {po.ChecksumCRC32, utils.HashTypeCRC32}, + {po.ChecksumCRC32C, utils.HashTypeCRC32C}, + {po.ChecksumSHA1, utils.HashTypeSha1}, + {po.ChecksumSHA256, utils.HashTypeSha256}, + {po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME}, + } + var hashRdr *utils.HashReader + + for _, config := range hashConfigs { + if config.value != nil { + hashRdr, err = utils.NewHashReader(rdr, *config.value, config.hashType) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } + } + + // If only the checksum algorithm is provided register + // a new HashReader to calculate the object checksum + if hashRdr == nil && po.ChecksumAlgorithm != "" { + hashRdr, err = utils.NewHashReader(rdr, "", utils.HashType(strings.ToLower(string(po.ChecksumAlgorithm)))) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } + _, err = io.Copy(f, rdr) if err != nil { if errors.Is(err, syscall.EDQUOT) { @@ -2351,6 +2810,38 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons } } + checksum := s3response.Checksum{} + + // Store the calculated checksum in the object metadata + if hashRdr != nil { + // The checksum type is always FULL_OBJECT for PutObject + checksum.Type = types.ChecksumTypeFullObject + + sum := hashRdr.Sum() + switch hashRdr.Type() { + case utils.HashTypeCRC32: + checksum.CRC32 = &sum + checksum.Algorithm = types.ChecksumAlgorithmCrc32 + case utils.HashTypeCRC32C: + checksum.CRC32C = &sum + checksum.Algorithm = types.ChecksumAlgorithmCrc32c + case utils.HashTypeSha1: + checksum.SHA1 = &sum + checksum.Algorithm = types.ChecksumAlgorithmSha1 + case utils.HashTypeSha256: + checksum.SHA256 = &sum + checksum.Algorithm = types.ChecksumAlgorithmSha256 + case utils.HashTypeCRC64NVME: + checksum.CRC64NVME = &sum + checksum.Algorithm = types.ChecksumAlgorithmCrc64nvme + } + + err := p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) + } + } + err = p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, etagkey, []byte(etag)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) @@ -2431,8 +2922,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons } return s3response.PutObjectOutput{ - ETag: etag, - VersionID: versionID, + ETag: etag, + VersionID: versionID, + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, + ChecksumCRC64NVME: checksum.CRC64NVME, + ChecksumType: checksum.Type, }, nil } @@ -2968,6 +3465,19 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO return nil, fmt.Errorf("open object: %w", err) } + var checksums s3response.Checksum + var cType types.ChecksumType + // Skip the checksums retreival if object isn't requested fully + if input.ChecksumMode == types.ChecksumModeEnabled && length-startOffset == objSize { + checksums, err = p.retrieveChecksums(f, bucket, object) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object checksums: %w", err) + } + if checksums.Type != "" { + cType = checksums.Type + } + } + // using an os.File allows zero-copy sendfile via io.Copy(os.File, net.Conn) var body io.ReadCloser = f if startOffset != 0 || length != objSize { @@ -2976,18 +3486,24 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO } return &s3.GetObjectOutput{ - AcceptRanges: &acceptRange, - ContentLength: &length, - ContentEncoding: &contentEncoding, - ContentType: &contentType, - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), - Metadata: userMetaData, - TagCount: tagCount, - ContentRange: &contentRange, - StorageClass: types.StorageClassStandard, - VersionId: &versionId, - Body: body, + AcceptRanges: &acceptRange, + ContentLength: &length, + ContentEncoding: &contentEncoding, + ContentType: &contentType, + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + Metadata: userMetaData, + TagCount: tagCount, + ContentRange: &contentRange, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, + Body: body, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumType: cType, }, nil } @@ -3161,6 +3677,18 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } } + var checksums s3response.Checksum + var cType types.ChecksumType + if input.ChecksumMode == types.ChecksumModeEnabled { + checksums, err = p.retrieveChecksums(nil, bucket, object) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object checksums: %w", err) + } + if checksums.Type != "" { + cType = checksums.Type + } + } + return &s3.HeadObjectOutput{ ContentLength: &size, ContentType: &contentType, @@ -3173,14 +3701,21 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ObjectLockRetainUntilDate: objectLockRetainUntilDate, StorageClass: types.StorageClassStandard, VersionId: &versionId, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumType: cType, }, nil } func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) { data, err := p.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: input.Bucket, - Key: input.Key, - VersionId: input.VersionId, + Bucket: input.Bucket, + Key: input.Key, + VersionId: input.VersionId, + ChecksumMode: types.ChecksumModeEnabled, }) if err != nil { if errors.Is(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) && data != nil { @@ -3200,6 +3735,14 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr LastModified: data.LastModified, VersionId: data.VersionId, DeleteMarker: data.DeleteMarker, + Checksum: &types.Checksum{ + ChecksumCRC32: data.ChecksumCRC32, + ChecksumCRC32C: data.ChecksumCRC32C, + ChecksumSHA1: data.ChecksumSHA1, + ChecksumSHA256: data.ChecksumSHA256, + ChecksumCRC64NVME: data.ChecksumCRC64NVME, + ChecksumType: data.ChecksumType, + }, }, nil } @@ -3296,6 +3839,11 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. var etag string var version *string + var crc32 *string + var crc32c *string + var sha1 *string + var sha256 *string + var crc64nvme *string dstObjdPath := filepath.Join(dstBucket, dstObject) if dstObjdPath == objPath { @@ -3318,6 +3866,59 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. } } + checksums, err := p.retrieveChecksums(nil, dstBucket, dstObject) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj checksums: %w", err) + } + + if input.ChecksumAlgorithm != "" { + // If a different checksum algorith is specified + // first caclculate and store the checksum + if checksums.Algorithm != input.ChecksumAlgorithm { + f, err := os.Open(dstObjdPath) + if err != nil { + return nil, fmt.Errorf("open obj file: %w", err) + } + defer f.Close() + + hashReader, err := utils.NewHashReader(f, "", utils.HashType(strings.ToLower(string(input.ChecksumAlgorithm)))) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + _, err = hashReader.Read(nil) + if err != nil { + return nil, fmt.Errorf("read err: %w", err) + } + + checksums = s3response.Checksum{} + + sum := hashReader.Sum() + switch hashReader.Type() { + case utils.HashTypeCRC32: + checksums.CRC32 = &sum + crc32 = &sum + case utils.HashTypeCRC32C: + checksums.CRC32C = &sum + crc32c = &sum + case utils.HashTypeSha1: + checksums.SHA1 = &sum + sha1 = &sum + case utils.HashTypeSha256: + checksums.SHA256 = &sum + sha256 = &sum + case utils.HashTypeCRC64NVME: + checksums.CRC64NVME = &sum + crc64nvme = &sum + } + + err = p.storeChecksums(f, dstBucket, dstObject, checksums) + if err != nil { + return nil, fmt.Errorf("store checksum: %w", err) + } + } + } + b, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, etagkey) etag = string(b) vId, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, versionIdKey) @@ -3327,19 +3928,37 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. version = backend.GetPtrFromString(string(vId)) } else { contentLength := fi.Size() + + checksums, err := p.retrieveChecksums(f, srcBucket, srcObject) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj checksum: %w", err) + } + + // If any checksum algorithm is provided, replace, otherwise + // use the existing one + if input.ChecksumAlgorithm != "" { + checksums.Algorithm = input.ChecksumAlgorithm + } + res, err := p.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &dstBucket, - Key: &dstObject, - Body: f, - ContentLength: &contentLength, - Metadata: input.Metadata, + Bucket: &dstBucket, + Key: &dstObject, + Body: f, + ContentLength: &contentLength, + Metadata: input.Metadata, + ChecksumAlgorithm: checksums.Algorithm, }) if err != nil { return nil, err } etag = res.ETag version = &res.VersionID + crc32 = res.ChecksumCRC32 + crc32c = res.ChecksumCRC32C + sha1 = res.ChecksumSHA1 + sha256 = res.ChecksumSHA256 + crc64nvme = res.ChecksumCRC64NVME } fi, err = os.Stat(dstObjdPath) @@ -3349,8 +3968,13 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. return &s3.CopyObjectOutput{ CopyObjectResult: &types.CopyObjectResult{ - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + ChecksumCRC32: crc32, + ChecksumCRC32C: crc32c, + ChecksumSHA1: sha1, + ChecksumSHA256: sha256, + ChecksumCRC64NVME: crc64nvme, }, VersionId: version, CopySourceVersionId: &srcVersionId, @@ -3447,6 +4071,12 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { return s3response.Object{}, backend.ErrSkipObj } + // Retreive the object checksum algorithm + checksums, err := p.retrieveChecksums(nil, bucket, path) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.Object{}, backend.ErrSkipObj + } + // file object, get object info and fill out object data etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) if errors.Is(err, fs.ErrNotExist) { @@ -3472,11 +4102,13 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { mtime := fi.ModTime() return s3response.Object{ - ETag: &etag, - Key: &path, - LastModified: &mtime, - Size: &size, - StorageClass: types.ObjectStorageClassStandard, + ETag: &etag, + Key: &path, + LastModified: &mtime, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{checksums.Algorithm}, + ChecksumType: checksums.Type, }, nil } } @@ -4107,6 +4739,25 @@ func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response. return buckets, nil } +func (p *Posix) storeChecksums(f *os.File, bucket, object string, chs s3response.Checksum) error { + checksums, err := json.Marshal(chs) + if err != nil { + return fmt.Errorf("parse checksum: %w", err) + } + + return p.meta.StoreAttribute(f, bucket, object, checksumsKey, checksums) +} + +func (p *Posix) retrieveChecksums(f *os.File, bucket, object string) (checksums s3response.Checksum, err error) { + checksumsAtr, err := p.meta.RetrieveAttribute(f, bucket, object, checksumsKey) + if err != nil { + return checksums, err + } + + err = json.Unmarshal(checksumsAtr, &checksums) + return checksums, err +} + func getString(str *string) string { if str == nil { return "" diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index 9f0b0b1f..fe9875dd 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -256,8 +256,10 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip ID: *u.Owner.ID, DisplayName: *u.Owner.DisplayName, }, - StorageClass: u.StorageClass, - Initiated: *u.Initiated, + StorageClass: u.StorageClass, + Initiated: *u.Initiated, + ChecksumAlgorithm: u.ChecksumAlgorithm, + ChecksumType: u.ChecksumType, }) } @@ -293,10 +295,15 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re var parts []s3response.Part for _, p := range output.Parts { parts = append(parts, s3response.Part{ - PartNumber: int(*p.PartNumber), - LastModified: *p.LastModified, - ETag: *p.ETag, - Size: *p.Size, + PartNumber: int(*p.PartNumber), + LastModified: *p.LastModified, + ETag: *p.ETag, + Size: *p.Size, + ChecksumCRC32: p.ChecksumCRC32, + ChecksumCRC32C: p.ChecksumCRC32C, + ChecksumCRC64NVME: p.ChecksumCRC64NVME, + ChecksumSHA1: p.ChecksumSHA1, + ChecksumSHA256: p.ChecksumSHA256, }) } pnm, err := strconv.Atoi(*output.PartNumberMarker) @@ -329,31 +336,34 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re MaxParts: int(*output.MaxParts), IsTruncated: *output.IsTruncated, Parts: parts, + ChecksumAlgorithm: output.ChecksumAlgorithm, + ChecksumType: output.ChecksumType, }, nil } -func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) { +func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { // streaming backend is not seekable, // use unsigned payload for streaming ops output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions( v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware, )) - if err != nil { - return "", handleError(err) - } - - return *output.ETag, nil + return output, handleError(err) } -func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { output, err := s.client.UploadPartCopy(ctx, input) if err != nil { - return s3response.CopyObjectResult{}, handleError(err) + return s3response.CopyPartResult{}, handleError(err) } - return s3response.CopyObjectResult{ - LastModified: *output.CopyPartResult.LastModified, - ETag: *output.CopyPartResult.ETag, + return s3response.CopyPartResult{ + LastModified: *output.CopyPartResult.LastModified, + ETag: output.CopyPartResult.ETag, + ChecksumCRC32: output.CopyPartResult.ChecksumCRC32, + ChecksumCRC32C: output.CopyPartResult.ChecksumCRC32C, + ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME, + ChecksumSHA1: output.CopyPartResult.ChecksumSHA1, + ChecksumSHA256: output.CopyPartResult.ChecksumSHA256, }, nil } @@ -373,8 +383,13 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3re } return s3response.PutObjectOutput{ - ETag: *output.ETag, - VersionID: versionID, + ETag: *output.ETag, + VersionID: versionID, + ChecksumCRC32: output.ChecksumCRC32, + ChecksumCRC32C: output.ChecksumCRC32C, + ChecksumCRC64NVME: output.ChecksumCRC64NVME, + ChecksumSHA1: output.ChecksumSHA1, + ChecksumSHA256: output.ChecksumSHA256, }, nil } @@ -425,6 +440,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt ObjectSize: out.ObjectSize, StorageClass: out.StorageClass, ObjectParts: &parts, + Checksum: out.Checksum, }, handleError(err) } @@ -837,13 +853,15 @@ func convertObjects(objs []types.Object) []s3response.Object { for _, obj := range objs { result = append(result, s3response.Object{ - ETag: obj.ETag, - Key: obj.Key, - LastModified: obj.LastModified, - Owner: obj.Owner, - Size: obj.Size, - RestoreStatus: obj.RestoreStatus, - StorageClass: obj.StorageClass, + ETag: obj.ETag, + Key: obj.Key, + LastModified: obj.LastModified, + Owner: obj.Owner, + Size: obj.Size, + RestoreStatus: obj.RestoreStatus, + StorageClass: obj.StorageClass, + ChecksumAlgorithm: obj.ChecksumAlgorithm, + ChecksumType: obj.ChecksumType, }) } diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 06124fa3..b29b2810 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -170,10 +170,10 @@ var _ backend.Backend = &BackendMock{} // StringFunc: func() string { // panic("mock out the String method") // }, -// UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) { +// UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) { // panic("mock out the UploadPart method") // }, -// UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +// UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { // panic("mock out the UploadPartCopy method") // }, // } @@ -331,10 +331,10 @@ type BackendMock struct { StringFunc func() string // UploadPartFunc mocks the UploadPart method. - UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) + UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) // UploadPartCopyFunc mocks the UploadPartCopy method. - UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) + UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) // calls tracks calls to the methods. calls struct { @@ -2620,7 +2620,7 @@ func (mock *BackendMock) StringCalls() []struct { } // UploadPart calls UploadPartFunc. -func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) { +func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) { if mock.UploadPartFunc == nil { panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called") } @@ -2656,7 +2656,7 @@ func (mock *BackendMock) UploadPartCalls() []struct { } // UploadPartCopy calls UploadPartCopyFunc. -func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { +func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { if mock.UploadPartCopyFunc == nil { panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called") } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 02947bf6..556be041 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -485,12 +485,27 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { }) } + checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + if c.debug { + log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode) + } + return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionGetObject, + BucketOwner: parsedAcl.Owner, + }) + } + ctx.Locals("logResBody", false) res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{ - Bucket: &bucket, - Key: &key, - Range: &acceptRange, - VersionId: &versionId, + Bucket: &bucket, + Key: &key, + Range: &acceptRange, + VersionId: &versionId, + ChecksumMode: checksumMode, }) if err != nil { if res != nil { @@ -568,6 +583,42 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Value: string(res.StorageClass), }) } + if res.ChecksumCRC32 != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc32", + Value: *res.ChecksumCRC32, + }) + } + if res.ChecksumCRC32C != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc32c", + Value: *res.ChecksumCRC32C, + }) + } + if res.ChecksumSHA1 != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-sha1", + Value: *res.ChecksumSHA1, + }) + } + if res.ChecksumSHA256 != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-sha256", + Value: *res.ChecksumSHA256, + }) + } + if res.ChecksumCRC64NVME != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc64nvme", + Value: *res.ChecksumCRC64NVME, + }) + } + if res.ChecksumType != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-type", + Value: string(res.ChecksumType), + }) + } // Set x-amz-meta-... headers utils.SetMetaHeaders(ctx, res.Metadata) @@ -1962,7 +2013,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { if c.debug { log.Printf("invalid part number: %d", partNumber) } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumber), &MetaOpts{ Logger: c.logger, MetricsMng: c.mm, @@ -2007,6 +2058,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { }) } + algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + return SendResponse(ctx, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }) + } + var body io.Reader bodyi := ctx.Locals("body-reader") if bodyi != nil { @@ -2016,16 +2078,59 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } ctx.Locals("logReqBody", false) - etag, err := c.be.UploadPart(ctx.Context(), + res, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{ - Bucket: &bucket, - Key: &keyStart, - UploadId: &uploadId, - PartNumber: &partNumber, - ContentLength: &contentLength, - Body: body, + Bucket: &bucket, + Key: &keyStart, + UploadId: &uploadId, + PartNumber: &partNumber, + ContentLength: &contentLength, + Body: body, + ChecksumAlgorithm: algorithm, + ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), }) - ctx.Response().Header.Set("Etag", etag) + if err == nil { + headers := []utils.CustomHeader{} + if res.ETag != nil { + headers = append(headers, utils.CustomHeader{ + Key: "ETag", + Value: *res.ETag, + }) + } + switch { + case res.ChecksumCRC32 != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc32", + Value: *res.ChecksumCRC32, + }) + case res.ChecksumCRC32C != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc32c", + Value: *res.ChecksumCRC32C, + }) + case res.ChecksumCRC64NVME != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc64nvme", + Value: *res.ChecksumCRC64NVME, + }) + case res.ChecksumSHA1 != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-sha1", + Value: *res.ChecksumSHA1, + }) + case res.ChecksumSHA256 != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-sha256", + Value: *res.ChecksumSHA256, + }) + } + + utils.SetResponseHeaders(ctx, headers) + } return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, @@ -2256,6 +2361,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { metaDirective = types.MetadataDirectiveReplace } + checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) + err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) + if err != nil { + if c.debug { + log.Printf("invalid checksum algorithm: %v", checksumAlgorithm) + } + return SendXMLResponse(ctx, nil, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.CopyObject(ctx.Context(), &s3.CopyObjectInput{ Bucket: &bucket, @@ -2269,6 +2389,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Metadata: metadata, MetadataDirective: metaDirective, StorageClass: types.StorageClass(storageClass), + ChecksumAlgorithm: checksumAlgorithm, }) if err == nil { hdrs := []utils.CustomHeader{} @@ -2368,6 +2489,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { }) } + algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + return SendResponse(ctx, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }) + } + var body io.Reader bodyi := ctx.Locals("body-reader") if bodyi != nil { @@ -2390,6 +2522,12 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ObjectLockRetainUntilDate: &objLock.RetainUntilDate, ObjectLockMode: objLock.ObjectLockMode, ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, + ChecksumAlgorithm: algorithm, + ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), }) if err != nil { return SendResponse(ctx, err, @@ -2417,6 +2555,39 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Value: res.VersionID, }) } + switch { + case res.ChecksumCRC32 != nil: + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc32", + Value: *res.ChecksumCRC32, + }) + case res.ChecksumCRC32C != nil: + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc32c", + Value: *res.ChecksumCRC32C, + }) + case res.ChecksumCRC64NVME != nil: + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc64nvme", + Value: *res.ChecksumCRC64NVME, + }) + case res.ChecksumSHA1 != nil: + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-sha1", + Value: *res.ChecksumSHA1, + }) + case res.ChecksumSHA256 != nil: + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-sha256", + Value: *res.ChecksumSHA256, + }) + } + if res.ChecksumType != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-type", + Value: string(res.ChecksumType), + }) + } utils.SetResponseHeaders(ctx, hdrs) @@ -2934,12 +3105,27 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { }) } + checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + if c.debug { + log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode) + } + return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.HeadObject(ctx.Context(), &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &key, - PartNumber: partNumber, - VersionId: &versionId, + Bucket: &bucket, + Key: &key, + PartNumber: partNumber, + VersionId: &versionId, + ChecksumMode: checksumMode, }) if err != nil { if res != nil { @@ -3022,6 +3208,39 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: string(res.StorageClass), }) } + switch { + case res.ChecksumCRC32 != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc32", + Value: *res.ChecksumCRC32, + }) + case res.ChecksumCRC32C != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc32c", + Value: *res.ChecksumCRC32C, + }) + case res.ChecksumCRC64NVME != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc64nvme", + Value: *res.ChecksumCRC64NVME, + }) + case res.ChecksumSHA1 != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-sha1", + Value: *res.ChecksumSHA1, + }) + case res.ChecksumSHA256 != nil: + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-sha256", + Value: *res.ChecksumSHA256, + }) + } + if res.ChecksumType != "" { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-type", + Value: string(res.ChecksumType), + }) + } contentType := getstring(res.ContentType) if contentType == "" { @@ -3232,6 +3451,35 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { }) } + _, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + if c.debug { + log.Printf("err parsing checksum headers: %v", err) + } + return SendXMLResponse(ctx, nil, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }) + } + + checksumType := types.ChecksumType(ctx.Get("x-amz-checksum-type")) + err = utils.IsChecksumTypeValid(checksumType) + if err != nil { + if c.debug { + log.Printf("invalid checksum type: %v", err) + } + return SendXMLResponse(ctx, nil, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.CompleteMultipartUpload(ctx.Context(), &s3.CompleteMultipartUploadInput{ Bucket: &bucket, @@ -3240,6 +3488,12 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { MultipartUpload: &types.CompletedMultipartUpload{ Parts: data.Parts, }, + ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumType: checksumType, }) if err == nil { if getstring(res.VersionId) != "" { @@ -3305,6 +3559,20 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { metadata := utils.GetUserMetaData(&ctx.Request().Header) + checksumAlgorithm, checksumType, err := utils.ParseCreateMpChecksumHeaders(ctx) + if err != nil { + if c.debug { + log.Printf("err parsing checksum headers: %v", err) + } + return SendXMLResponse(ctx, nil, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCreateMultipartUpload, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.CreateMultipartUpload(ctx.Context(), &s3.CreateMultipartUploadInput{ Bucket: &bucket, @@ -3316,7 +3584,19 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { ObjectLockMode: objLockState.ObjectLockMode, ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus, Metadata: metadata, + ChecksumAlgorithm: checksumAlgorithm, + ChecksumType: checksumType, }) + if err == nil { + if checksumAlgorithm != "" { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-checksum-algorithm", + Value: string(checksumAlgorithm), + }, + }) + } + } return SendXMLResponse(ctx, res, err, &MetaOpts{ Logger: c.logger, diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 6546237a..14a5be6d 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -232,6 +232,9 @@ func TestS3ApiController_GetActions(t *testing.T) { getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil) getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello") + invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil) + invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode") + tests := []struct { name string app *fiber.App @@ -329,6 +332,15 @@ func TestS3ApiController_GetActions(t *testing.T) { wantErr: false, statusCode: 200, }, + { + name: "Get-actions-get-object-invalid-checksum-mode", + app: app, + args: args{ + req: invalidChecksumMode, + }, + wantErr: false, + statusCode: 400, + }, { name: "Get-actions-get-object-success", app: app, @@ -971,14 +983,14 @@ func TestS3ApiController_PutActions(t *testing.T) { PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { return s3response.PutObjectOutput{}, nil }, - UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) { - return "hello", nil + UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + return &s3.UploadPartOutput{}, nil }, PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error { return nil }, - UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { - return s3response.CopyObjectResult{}, nil + UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { + return s3response.CopyPartResult{}, nil }, PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error { return nil @@ -1012,6 +1024,11 @@ func TestS3ApiController_PutActions(t *testing.T) { cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") + // CopyObject invalid checksum algorithm + cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") + cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") + // PutObjectAcl success aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) aclReq.Header.Set("X-Amz-Acl", "private") @@ -1033,6 +1050,40 @@ func TestS3ApiController_PutActions(t *testing.T) { invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body)) invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello") + // PutObject invalid checksum algorithm + invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") + + // PutObject invalid base64 checksum + invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64") + + // PutObject invalid crc32 + invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm") + + // PutObject invalid crc32c + invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==") + + // PutObject invalid sha1 + invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==") + + // PutObject invalid sha256 + invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm") + + // PutObject multiple checksum headers + mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=") + mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==") + + // PutObject checksum algorithm and header mismatch + checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) + checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1") + checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==") + tests := []struct { name string app *fiber.App @@ -1175,6 +1226,15 @@ func TestS3ApiController_PutActions(t *testing.T) { wantErr: false, statusCode: 200, }, + { + name: "Copy-object-invalid-checksum-algorithm", + app: app, + args: args{ + req: cpyInvChecksumAlgo, + }, + wantErr: false, + statusCode: 400, + }, { name: "Copy-object-success", app: app, @@ -1642,6 +1702,9 @@ func TestS3ApiController_HeadObject(t *testing.T) { }) appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject) + invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil) + invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode") + tests := []struct { name string app *fiber.App @@ -1658,6 +1721,15 @@ func TestS3ApiController_HeadObject(t *testing.T) { wantErr: false, statusCode: 200, }, + { + name: "Head-object-invalid-checksum-mode", + app: app, + args: args{ + req: invChecksumMode, + }, + wantErr: false, + statusCode: 400, + }, { name: "Head-object-error", app: appErr, @@ -1735,6 +1807,9 @@ func TestS3ApiController_CreateActions(t *testing.T) { }) app.Post("/:bucket/:key/*", s3ApiController.CreateActions) + invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil) + invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") + tests := []struct { name string app *fiber.App @@ -1796,6 +1871,15 @@ func TestS3ApiController_CreateActions(t *testing.T) { wantErr: false, statusCode: 200, }, + { + name: "Create-multipart-upload-invalid-checksum-algorithm", + app: app, + args: args{ + req: invChecksumAlgo, + }, + wantErr: false, + statusCode: 400, + }, { name: "Create-multipart-upload-success", app: app, diff --git a/s3api/middlewares/md5.go b/s3api/middlewares/md5.go index 8dd5740d..e1b81c65 100644 --- a/s3api/middlewares/md5.go +++ b/s3api/middlewares/md5.go @@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler { } sum := md5.Sum(ctx.Body()) - calculatedSum := utils.Md5SumString(sum[:]) + calculatedSum := utils.Base64SumString(sum[:]) if incomingSum != calculatedSum { return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger}) diff --git a/s3api/utils/auth-reader.go b/s3api/utils/auth-reader.go index 2bbdce39..990e5a33 100644 --- a/s3api/utils/auth-reader.go +++ b/s3api/utils/auth-reader.go @@ -56,7 +56,7 @@ func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, de var hr *HashReader hashPayload := ctx.Get("X-Amz-Content-Sha256") if !IsSpecialPayload(hashPayload) { - hr, _ = NewHashReader(r, "", HashTypeSha256) + hr, _ = NewHashReader(r, "", HashTypeSha256Hex) } else { hr, _ = NewHashReader(r, "", HashTypeNone) } diff --git a/s3api/utils/csum-reader.go b/s3api/utils/csum-reader.go index 04ce90d7..ce2e35e7 100644 --- a/s3api/utils/csum-reader.go +++ b/s3api/utils/csum-reader.go @@ -16,13 +16,19 @@ package utils import ( "crypto/md5" + "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/hex" "errors" + "fmt" "hash" + "hash/crc32" + "hash/crc64" "io" + "math/bits" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/s3err" ) @@ -31,11 +37,21 @@ type HashType string const ( // HashTypeMd5 generates MD5 checksum for the data stream - HashTypeMd5 = "md5" - // HashTypeSha256 generates SHA256 checksum for the data stream - HashTypeSha256 = "sha256" + HashTypeMd5 HashType = "md5" + // HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream + HashTypeSha256 HashType = "sha256" + // HashTypeSha256Hex generates SHA256 hex encoded checksum for the data stream + HashTypeSha256Hex HashType = "sha256-hex" + // HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream + HashTypeSha1 HashType = "sha1" + // HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream + HashTypeCRC32 HashType = "crc32" + // HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream + HashTypeCRC32C HashType = "crc32c" + // HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream + HashTypeCRC64NVME HashType = "crc64nvme" // HashTypeNone is a no-op checksum for the data stream - HashTypeNone = "none" + HashTypeNone HashType = "none" ) // HashReader is an io.Reader that calculates the checksum @@ -62,8 +78,18 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e switch ht { case HashTypeMd5: hash = md5.New() + case HashTypeSha256Hex: + hash = sha256.New() case HashTypeSha256: hash = sha256.New() + case HashTypeSha1: + hash = sha1.New() + case HashTypeCRC32: + hash = crc32.NewIEEE() + case HashTypeCRC32C: + hash = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case HashTypeCRC64NVME: + hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))) case HashTypeNone: hash = noop{} default: @@ -88,15 +114,40 @@ func (hr *HashReader) Read(p []byte) (int, error) { if errors.Is(readerr, io.EOF) && hr.sum != "" { switch hr.hashType { case HashTypeMd5: - sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil)) + sum := hr.Sum() if sum != hr.sum { return n, s3err.GetAPIError(s3err.ErrInvalidDigest) } - case HashTypeSha256: - sum := hex.EncodeToString(hr.hash.Sum(nil)) + case HashTypeSha256Hex: + sum := hr.Sum() if sum != hr.sum { return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch) } + case HashTypeCRC32: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32) + } + case HashTypeCRC32C: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32c) + } + case HashTypeSha1: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha1) + } + case HashTypeSha256: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256) + } + case HashTypeCRC64NVME: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme) + } default: return n, errInvalidHashType } @@ -104,20 +155,38 @@ func (hr *HashReader) Read(p []byte) (int, error) { return n, readerr } +func (hr *HashReader) SetReader(r io.Reader) { + hr.r = r +} + // Sum returns the checksum hash of the data read so far func (hr *HashReader) Sum() string { switch hr.hashType { case HashTypeMd5: - return Md5SumString(hr.hash.Sum(nil)) - case HashTypeSha256: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeSha256Hex: return hex.EncodeToString(hr.hash.Sum(nil)) + case HashTypeCRC32: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeCRC32C: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeSha1: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeSha256: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeCRC64NVME: + return Base64SumString(hr.hash.Sum(nil)) default: return "" } } +func (hr *HashReader) Type() HashType { + return hr.hashType +} + // Md5SumString converts the hash bytes to the string checksum value -func Md5SumString(b []byte) string { +func Base64SumString(b []byte) string { return base64.StdEncoding.EncodeToString(b) } @@ -128,3 +197,59 @@ func (n noop) Sum(b []byte) []byte { return []byte{} } func (n noop) Reset() {} func (n noop) Size() int { return 0 } func (n noop) BlockSize() int { return 1 } + +// NewCompositeChecksumReader initializes a composite checksum +// processor, which decodes and validates the provided +// checksums and returns the final checksum based on +// the previous processings. +// +// The supported checksum types are: +// - CRC32 +// - CRC32C +// - SHA1 +// - SHA256 +func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) { + var hasher hash.Hash + switch ht { + case HashTypeSha256: + hasher = sha256.New() + case HashTypeSha1: + hasher = sha1.New() + case HashTypeCRC32: + hasher = crc32.NewIEEE() + case HashTypeCRC32C: + hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case HashTypeNone: + hasher = noop{} + default: + return nil, errInvalidHashType + } + + return &CompositeChecksumReader{ + hasher: hasher, + }, nil +} + +type CompositeChecksumReader struct { + hasher hash.Hash +} + +// Decodes and writes the checksum in the hasher +func (ccr *CompositeChecksumReader) Process(checksum string) error { + data, err := base64.StdEncoding.DecodeString(checksum) + if err != nil { + return fmt.Errorf("base64 decode: %w", err) + } + + _, err = ccr.hasher.Write(data) + if err != nil { + return fmt.Errorf("hash write: %w", err) + } + + return nil +} + +// Returns the base64 encoded composite checksum +func (ccr *CompositeChecksumReader) Sum() string { + return Base64SumString(ccr.hasher.Sum(nil)) +} diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index fe95285c..c1ace5b1 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -16,6 +16,7 @@ package utils import ( "bytes" + "encoding/base64" "errors" "fmt" "io" @@ -296,7 +297,9 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok { output.StorageClass = "" } - fmt.Printf("%+v\n", output) + if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok { + output.Checksum = nil + } return output } @@ -456,3 +459,170 @@ func shouldEscape(c byte) bool { return true } + +func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) { + sdkAlgorithm := types.ChecksumAlgorithm(ctx.Get("X-Amz-Sdk-Checksum-Algorithm")) + + err := IsChecksumAlgorithmValid(sdkAlgorithm) + if err != nil { + return "", nil, err + } + + checksums := map[types.ChecksumAlgorithm]string{ + types.ChecksumAlgorithmCrc32: ctx.Get("X-Amz-Checksum-Crc32"), + types.ChecksumAlgorithmCrc32c: ctx.Get("X-Amz-Checksum-Crc32c"), + types.ChecksumAlgorithmSha1: ctx.Get("X-Amz-Checksum-Sha1"), + types.ChecksumAlgorithmSha256: ctx.Get("X-Amz-Checksum-Sha256"), + types.ChecksumAlgorithmCrc64nvme: ctx.Get("X-Amz-Checksum-Crc64nvme"), + } + + headerCtr := 0 + + for al, val := range checksums { + if val != "" && !IsValidChecksum(val, al) { + return sdkAlgorithm, checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al)))) + } + // If any other checksum value is provided, + // rather than x-amz-sdk-checksum-algorithm + if sdkAlgorithm != "" && sdkAlgorithm != al && val != "" { + return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders) + } + if val != "" { + sdkAlgorithm = al + headerCtr++ + } + + if headerCtr > 1 { + return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders) + } + } + + return sdkAlgorithm, checksums, nil +} + +var checksumLengths = map[types.ChecksumAlgorithm]int{ + types.ChecksumAlgorithmCrc32: 4, + types.ChecksumAlgorithmCrc32c: 4, + types.ChecksumAlgorithmCrc64nvme: 8, + types.ChecksumAlgorithmSha1: 20, + types.ChecksumAlgorithmSha256: 32, +} + +func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool { + decoded, err := base64.StdEncoding.DecodeString(checksum) + if err != nil { + return false + } + + expectedLength, exists := checksumLengths[algorithm] + if !exists { + return false + } + + return len(decoded) == expectedLength +} + +func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error { + if alg != "" && + alg != types.ChecksumAlgorithmCrc32 && + alg != types.ChecksumAlgorithmCrc32c && + alg != types.ChecksumAlgorithmSha1 && + alg != types.ChecksumAlgorithmSha256 && + alg != types.ChecksumAlgorithmCrc64nvme { + return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm) + } + + return nil +} + +// Validates the provided checksum type +func IsChecksumTypeValid(t types.ChecksumType) error { + if t != "" && + t != types.ChecksumTypeComposite && + t != types.ChecksumTypeFullObject { + return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type") + } + return nil +} + +type checksumTypeSchema map[types.ChecksumType]struct{} +type checksumSchema map[types.ChecksumAlgorithm]checksumTypeSchema + +// A table defining the checksum algorithm/type support +var checksumMap checksumSchema = checksumSchema{ + types.ChecksumAlgorithmCrc32: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + types.ChecksumTypeFullObject: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmCrc32c: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + types.ChecksumTypeFullObject: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmSha1: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmSha256: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmCrc64nvme: checksumTypeSchema{ + types.ChecksumTypeFullObject: struct{}{}, + "": struct{}{}, + }, + // Both could be empty + "": checksumTypeSchema{ + "": struct{}{}, + }, +} + +// Checks if checksum type and algorithm are supported together +func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType) error { + typeSchema := checksumMap[algo] + _, ok := typeSchema[t] + if !ok { + return s3err.GetChecksumSchemaMismatchErr(algo, t) + } + + return nil +} + +// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers +func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) { + algo := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) + if err := IsChecksumAlgorithmValid(algo); err != nil { + return "", "", err + } + + chType := types.ChecksumType(ctx.Get("x-amz-checksum-type")) + if err := IsChecksumTypeValid(chType); err != nil { + return "", "", err + } + + // Verify if checksum algorithm is provided, if + // checksum type is specified + if chType != "" && algo == "" { + return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo) + } + + // Verify if the checksum type is supported for + // the provided checksum algorithm + if err := checkChecksumTypeAndAlgo(algo, chType); err != nil { + return algo, chType, err + } + + // x-amz-checksum-type defaults to COMPOSITE + // if x-amz-checksum-algorithm is set except + // for the CRC64NVME algorithm: it defaults to FULL_OBJECT + if algo != "" && chType == "" { + if algo == types.ChecksumAlgorithmCrc64nvme { + chType = types.ChecksumTypeFullObject + } else { + chType = types.ChecksumTypeComposite + } + } + + return algo, chType, nil +} diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 1f767592..6aa09078 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -527,3 +527,325 @@ func Test_escapePath(t *testing.T) { }) } } + +func TestIsChecksumAlgorithmValid(t *testing.T) { + type args struct { + alg types.ChecksumAlgorithm + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "empty", + args: args{ + alg: "", + }, + wantErr: false, + }, + { + name: "crc32", + args: args{ + alg: types.ChecksumAlgorithmCrc32, + }, + wantErr: false, + }, + { + name: "crc32c", + args: args{ + alg: types.ChecksumAlgorithmCrc32c, + }, + wantErr: false, + }, + { + name: "sha1", + args: args{ + alg: types.ChecksumAlgorithmSha1, + }, + wantErr: false, + }, + { + name: "sha256", + args: args{ + alg: types.ChecksumAlgorithmSha256, + }, + wantErr: false, + }, + { + name: "crc64nvme", + args: args{ + alg: types.ChecksumAlgorithmCrc64nvme, + }, + wantErr: false, + }, + { + name: "invalid", + args: args{ + alg: types.ChecksumAlgorithm("invalid_checksum_algorithm"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := IsChecksumAlgorithmValid(tt.args.alg); (err != nil) != tt.wantErr { + t.Errorf("IsChecksumAlgorithmValid() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestIsValidChecksum(t *testing.T) { + type args struct { + checksum string + algorithm types.ChecksumAlgorithm + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "invalid-base64", + args: args{ + checksum: "invalid_base64_string", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: false, + }, + { + name: "invalid-crc32", + args: args{ + checksum: "YXNkZmFzZGZhc2Rm", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: false, + }, + { + name: "valid-crc32", + args: args{ + checksum: "ww2FVQ==", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: true, + }, + { + name: "invalid-crc32c", + args: args{ + checksum: "Zmdoa2doZmtnZmhr", + algorithm: types.ChecksumAlgorithmCrc32c, + }, + want: false, + }, + { + name: "valid-crc32c", + args: args{ + checksum: "DOsb4w==", + algorithm: types.ChecksumAlgorithmCrc32c, + }, + want: true, + }, + { + name: "invalid-sha1", + args: args{ + checksum: "YXNkZmFzZGZhc2RmYXNkZnNhZGZzYWRm", + algorithm: types.ChecksumAlgorithmSha1, + }, + want: false, + }, + { + name: "valid-sha1", + args: args{ + checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=", + algorithm: types.ChecksumAlgorithmSha1, + }, + want: true, + }, + { + name: "invalid-sha256", + args: args{ + checksum: "Zmdoa2doZmtnZmhrYXNkZmFzZGZhc2RmZHNmYXNkZg==", + algorithm: types.ChecksumAlgorithmSha256, + }, + want: false, + }, + { + name: "valid-sha256", + args: args{ + checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=", + algorithm: types.ChecksumAlgorithmSha256, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidChecksum(tt.args.checksum, tt.args.algorithm); got != tt.want { + t.Errorf("IsValidChecksum() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsChecksumTypeValid(t *testing.T) { + type args struct { + t types.ChecksumType + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "valid_FULL_OBJECT", + args: args{ + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "valid_COMPOSITE", + args: args{ + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "invalid", + args: args{ + t: types.ChecksumType("invalid_checksum_type"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := IsChecksumTypeValid(tt.args.t); (err != nil) != tt.wantErr { + t.Errorf("IsChecksumTypeValid() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_checkChecksumTypeAndAlgo(t *testing.T) { + type args struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "full_object-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-crc32c", + args: args{ + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-sha1", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-sha256", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-crc64nvme", + args: args{ + algo: types.ChecksumAlgorithmCrc64nvme, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "composite-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-crc32c", + args: args{ + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-sha1", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-sha256", + args: args{ + algo: types.ChecksumAlgorithmSha256, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-crc64nvme", + args: args{ + algo: types.ChecksumAlgorithmCrc64nvme, + t: types.ChecksumTypeComposite, + }, + wantErr: true, + }, + { + name: "composite-empty", + args: args{ + t: types.ChecksumTypeComposite, + }, + wantErr: true, + }, + { + name: "full_object-empty", + args: args{ + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := checkChecksumTypeAndAlgo(tt.args.algo, tt.args.t); (err != nil) != tt.wantErr { + t.Errorf("checkChecksumTypeAndAlgo() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/s3err/s3err.go b/s3err/s3err.go index 3db27633..e0e96216 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -17,7 +17,11 @@ package s3err import ( "bytes" "encoding/xml" + "fmt" "net/http" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3/types" ) // APIError structure @@ -74,6 +78,8 @@ const ( ErrInvalidPart ErrEmptyParts ErrInvalidPartNumber + ErrInvalidPartOrder + ErrInvalidCompleteMpPartNumber ErrInternalError ErrInvalidCopyDest ErrInvalidCopySource @@ -140,6 +146,10 @@ const ( ErrInvalidVersionId ErrNoSuchVersion ErrSuspendedVersioningNotAllowed + ErrMultipleChecksumHeaders + ErrInvalidChecksumAlgorithm + ErrInvalidChecksumPart + ErrChecksumTypeWithAlgo // Non-AWS errors ErrExistingObjectIsDirectory @@ -264,6 +274,16 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Part number must be an integer between 1 and 10000, inclusive.", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidPartOrder: { + Code: "InvalidPartOrder", + Description: "The list of parts was not in ascending order. Parts must be ordered by part number.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidCompleteMpPartNumber: { + Code: "InvalidArgument", + Description: "PartNumber must be >= 1", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidCopyDest: { Code: "InvalidRequest", Description: "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.", @@ -584,6 +604,26 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.", HTTPStatusCode: http.StatusBadRequest, }, + ErrMultipleChecksumHeaders: { + Code: "InvalidRequest", + Description: "Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidChecksumAlgorithm: { + Code: "InvalidRequest", + Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidChecksumPart: { + Code: "InvalidArgument", + Description: "Invalid Base64 or multiple checksums present in request", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrChecksumTypeWithAlgo: { + Code: "InvalidRequest", + Description: "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.", + HTTPStatusCode: http.StatusBadRequest, + }, // non aws errors ErrExistingObjectIsDirectory: { @@ -678,3 +718,48 @@ func encodeResponse(response interface{}) []byte { e.Encode(response) return bytesBuffer.Bytes() } + +// Returns invalid checksum error with the provided header in the error description +func GetInvalidChecksumHeaderErr(header string) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("Value for %v header is invalid.", header), + HTTPStatusCode: http.StatusBadRequest, + } +} + +// Returns checksum type mismatch APIError +func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("Checksum Type mismatch occurred, expected checksum Type: %v, actual checksum Type: %v", expected, actual), + HTTPStatusCode: http.StatusBadRequest, + } +} + +// Returns incorrect checksum APIError +func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError { + return APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The %v you specified did not match the calculated checksum.", strings.ToLower(string(algo))), + HTTPStatusCode: http.StatusBadRequest, + } +} + +// Returns checksum type mismatch error with checksum algorithm +func GetChecksumSchemaMismatchErr(algo types.ChecksumAlgorithm, t types.ChecksumType) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The %v checksum type cannot be used with the %v checksum algorithm.", algo, strings.ToLower(string(t))), + HTTPStatusCode: http.StatusBadRequest, + } +} + +// Returns checksum type mismatch error for multipart uploads +func GetChecksumTypeMismatchOnMpErr(t types.ChecksumType) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The upload was created using the %v checksum mode. The complete request must use the same checksum mode.", t), + HTTPStatusCode: http.StatusBadRequest, + } +} diff --git a/s3response/s3response.go b/s3response/s3response.go index 310d3efb..16603928 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -29,16 +29,27 @@ const ( ) type PutObjectOutput struct { - ETag string - VersionID string + ETag string + VersionID string + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string + ChecksumCRC64NVME *string + ChecksumType types.ChecksumType } // Part describes part metadata. type Part struct { - PartNumber int - LastModified time.Time - ETag string - Size int64 + PartNumber int + LastModified time.Time + ETag string + Size int64 + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string + ChecksumCRC64NVME *string } func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -59,9 +70,11 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type ListPartsResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"` - Bucket string - Key string - UploadID string `xml:"UploadId"` + Bucket string + Key string + UploadID string `xml:"UploadId"` + ChecksumAlgorithm types.ChecksumAlgorithm + ChecksumType types.ChecksumType Initiator Initiator Owner Owner @@ -101,6 +114,7 @@ type GetObjectAttributesResponse struct { ObjectSize *int64 StorageClass types.StorageClass `xml:",omitempty"` ObjectParts *ObjectParts + Checksum *types.Checksum // Not included in the response body VersionId *string @@ -169,13 +183,15 @@ type ListObjectsV2Result struct { } type Object struct { - ETag *string - Key *string - LastModified *time.Time - Owner *types.Owner - RestoreStatus *types.RestoreStatus - Size *int64 - StorageClass types.ObjectStorageClass + ChecksumAlgorithm []types.ChecksumAlgorithm + ChecksumType types.ChecksumType + ETag *string + Key *string + LastModified *time.Time + Owner *types.Owner + RestoreStatus *types.RestoreStatus + Size *int64 + StorageClass types.ObjectStorageClass } func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -197,12 +213,14 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error { // Upload describes in progress multipart upload type Upload struct { - Key string - UploadID string `xml:"UploadId"` - Initiator Initiator - Owner Owner - StorageClass types.StorageClass - Initiated time.Time + Key string + UploadID string `xml:"UploadId"` + Initiator Initiator + Owner Owner + StorageClass types.StorageClass + Initiated time.Time + ChecksumAlgorithm types.ChecksumAlgorithm + ChecksumType types.ChecksumType } func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -332,6 +350,20 @@ type CopyObjectResult struct { CopySourceVersionId string `xml:"-"` } +type CopyPartResult struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"` + LastModified time.Time + ETag *string + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string + ChecksumCRC64NVME *string + + // not included in the body + CopySourceVersionId string `xml:"-"` +} + func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type Alias CopyObjectResult aux := &struct { @@ -467,3 +499,14 @@ func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) { type ListBucketsResult struct { Buckets []Bucket } + +type Checksum struct { + Algorithm types.ChecksumAlgorithm + Type types.ChecksumType + + CRC32 *string + CRC32C *string + SHA1 *string + SHA256 *string + CRC64NVME *string +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 282da647..f94dd40f 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -138,6 +138,14 @@ func TestPutObject(s *S3Conf) { PutObject_invalid_long_tags(s) PutObject_missing_object_lock_retention_config(s) PutObject_with_object_lock(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + PutObject_checksum_algorithm_and_header_mismatch(s) + PutObject_multiple_checksum_headers(s) + PutObject_invalid_checksum_header(s) + PutObject_incorrect_checksums(s) + PutObject_checksums_success(s) + } PutObject_success(s) if !s.versioningEnabled { PutObject_racey_success(s) @@ -154,6 +162,11 @@ func TestHeadObject(s *S3Conf) { HeadObject_non_existing_dir_object(s) HeadObject_with_contenttype(s) HeadObject_invalid_parent_dir(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + HeadObject_not_enabled_checksum_mode(s) + HeadObject_checksums(s) + } HeadObject_success(s) } @@ -164,6 +177,11 @@ func TestGetObjectAttributes(s *S3Conf) { GetObjectAttributes_invalid_parent(s) GetObjectAttributes_empty_attrs(s) GetObjectAttributes_existing_object(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + + GetObjectAttributes_checksums(s) + } } func TestGetObject(s *S3Conf) { @@ -173,6 +191,10 @@ func TestGetObject(s *S3Conf) { GetObject_invalid_parent(s) GetObject_with_meta(s) GetObject_large_object(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + GetObject_checksums(s) + } GetObject_success(s) GetObject_directory_success(s) GetObject_by_range_success(s) @@ -191,6 +213,10 @@ func TestListObjects(s *S3Conf) { ListObjects_max_keys_none(s) ListObjects_marker_not_from_obj_list(s) ListObjects_list_all_objs(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + ListObjects_with_checksum(s) + } } func TestListObjectsV2(s *S3Conf) { @@ -203,6 +229,10 @@ func TestListObjectsV2(s *S3Conf) { ListObjectsV2_truncated_common_prefixes(s) ListObjectsV2_all_objs_max_keys(s) ListObjectsV2_list_all_objs(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + ListObjectsV2_with_checksum(s) + } ListObjectsV2_invalid_parent_prefix(s) } @@ -233,6 +263,14 @@ func TestCopyObject(s *S3Conf) { CopyObject_to_itself_with_new_metadata(s) CopyObject_CopySource_starting_with_slash(s) CopyObject_non_existing_dir_object(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + CopyObject_invalid_checksum_algorithm(s) + CopyObject_create_checksum_on_copy(s) + CopyObject_should_copy_the_existing_checksum(s) + CopyObject_should_replace_the_existing_checksum(s) + CopyObject_to_itself_by_replacing_the_checksum(s) + } CopyObject_success(s) } @@ -265,6 +303,13 @@ func TestCreateMultipartUpload(s *S3Conf) { CreateMultipartUpload_with_object_lock_not_enabled(s) CreateMultipartUpload_with_object_lock_invalid_retention(s) CreateMultipartUpload_past_retain_until_date(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + CreateMultipartUpload_invalid_checksum_algorithm(s) + CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type(s) + CreateMultipartUpload_invalid_checksum_type(s) + CreateMultipartUpload_valid_checksum_algorithm(s) + } CreateMultipartUpload_success(s) } @@ -273,6 +318,16 @@ func TestUploadPart(s *S3Conf) { UploadPart_invalid_part_number(s) UploadPart_non_existing_key(s) UploadPart_non_existing_mp_upload(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + UploadPart_checksum_algorithm_and_header_mismatch(s) + UploadPart_multiple_checksum_headers(s) + UploadPart_invalid_checksum_header(s) + UploadPart_checksum_algorithm_mistmatch_on_initialization(s) + UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s) + UploadPart_incorrect_checksums(s) + UploadPart_with_checksums_success(s) + } UploadPart_success(s) } @@ -288,6 +343,12 @@ func TestUploadPartCopy(s *S3Conf) { UploadPartCopy_by_range_invalid_range(s) UploadPartCopy_greater_range_than_obj_size(s) UploadPartCopy_by_range_success(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + UploadPartCopy_should_copy_the_checksum(s) + UploadPartCopy_should_not_copy_the_checksum(s) + UploadPartCopy_should_calculate_the_checksum(s) + } } func TestListParts(s *S3Conf) { @@ -296,6 +357,10 @@ func TestListParts(s *S3Conf) { ListParts_invalid_max_parts(s) ListParts_default_max_parts(s) ListParts_truncated(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + ListParts_with_checksums(s) + } ListParts_success(s) } @@ -306,6 +371,10 @@ func TestListMultipartUploads(s *S3Conf) { ListMultipartUploads_max_uploads(s) ListMultipartUploads_incorrect_next_key_marker(s) ListMultipartUploads_ignore_upload_id_marker(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + ListMultipartUploads_with_checksums(s) + } ListMultipartUploads_success(s) } @@ -319,10 +388,28 @@ func TestAbortMultipartUpload(s *S3Conf) { func TestCompleteMultipartUpload(s *S3Conf) { CompletedMultipartUpload_non_existing_bucket(s) + CompleteMultipartUpload_incorrect_part_number(s) CompleteMultipartUpload_invalid_part_number(s) CompleteMultipartUpload_invalid_ETag(s) CompleteMultipartUpload_small_upload_size(s) CompleteMultipartUpload_empty_parts(s) + CompleteMultipartUpload_incorrect_parts_order(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + CompleteMultipartUpload_invalid_checksum_type(s) + CompleteMultipartUpload_invalid_checksum_part(s) + CompleteMultipartUpload_multiple_checksum_part(s) + CompleteMultipartUpload_incorrect_checksum_part(s) + CompleteMultipartUpload_different_checksum_part(s) + CompleteMultipartUpload_missing_part_checksum(s) + CompleteMultipartUpload_multiple_final_checksums(s) + CompleteMultipartUpload_invalid_final_checksums(s) + CompleteMultipartUpload_incorrect_final_checksums(s) + CompleteMultipartUpload_should_calculate_the_final_checksum_full_object(s) + CompleteMultipartUpload_should_verify_the_final_checksum(s) + CompleteMultipartUpload_checksum_type_mismatch(s) + CompleteMultipartUpload_should_ignore_the_final_checksum(s) + } CompleteMultipartUpload_success(s) if !s.azureTests { CompleteMultipartUpload_racey_success(s) @@ -644,400 +731,444 @@ type IntTests map[string]func(s *S3Conf) error func GetIntTests() IntTests { return IntTests{ - "Authentication_empty_auth_header": Authentication_empty_auth_header, - "Authentication_invalid_auth_header": Authentication_invalid_auth_header, - "Authentication_unsupported_signature_version": Authentication_unsupported_signature_version, - "Authentication_malformed_credentials": Authentication_malformed_credentials, - "Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts, - "Authentication_credentials_terminated_string": Authentication_credentials_terminated_string, - "Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service, - "Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region, - "Authentication_credentials_invalid_date": Authentication_credentials_invalid_date, - "Authentication_credentials_future_date": Authentication_credentials_future_date, - "Authentication_credentials_past_date": Authentication_credentials_past_date, - "Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key, - "Authentication_invalid_signed_headers": Authentication_invalid_signed_headers, - "Authentication_missing_date_header": Authentication_missing_date_header, - "Authentication_invalid_date_header": Authentication_invalid_date_header, - "Authentication_date_mismatch": Authentication_date_mismatch, - "Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash, - "Authentication_incorrect_md5": Authentication_incorrect_md5, - "Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key, - "PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param, - "PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm, - "PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param, - "PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts, - "PresignedAuth_creds_invalid_terminator": PresignedAuth_creds_invalid_terminator, - "PresignedAuth_creds_incorrect_service": PresignedAuth_creds_incorrect_service, - "PresignedAuth_creds_incorrect_region": PresignedAuth_creds_incorrect_region, - "PresignedAuth_creds_invalid_date": PresignedAuth_creds_invalid_date, - "PresignedAuth_missing_date_query": PresignedAuth_missing_date_query, - "PresignedAuth_dates_mismatch": PresignedAuth_dates_mismatch, - "PresignedAuth_non_existing_access_key_id": PresignedAuth_non_existing_access_key_id, - "PresignedAuth_missing_signed_headers_query_param": PresignedAuth_missing_signed_headers_query_param, - "PresignedAuth_missing_expiration_query_param": PresignedAuth_missing_expiration_query_param, - "PresignedAuth_invalid_expiration_query_param": PresignedAuth_invalid_expiration_query_param, - "PresignedAuth_negative_expiration_query_param": PresignedAuth_negative_expiration_query_param, - "PresignedAuth_exceeding_expiration_query_param": PresignedAuth_exceeding_expiration_query_param, - "PresignedAuth_expired_request": PresignedAuth_expired_request, - "PresignedAuth_incorrect_secret_key": PresignedAuth_incorrect_secret_key, - "PresignedAuth_PutObject_success": PresignedAuth_PutObject_success, - "PutObject_missing_object_lock_retention_config": PutObject_missing_object_lock_retention_config, - "PutObject_name_too_long": PutObject_name_too_long, - "PutObject_with_object_lock": PutObject_with_object_lock, - "PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data, - "PresignedAuth_Put_GetObject_with_UTF8_chars": PresignedAuth_Put_GetObject_with_UTF8_chars, - "PresignedAuth_UploadPart": PresignedAuth_UploadPart, - "CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name, - "CreateBucket_existing_bucket": CreateBucket_existing_bucket, - "CreateBucket_owned_by_you": CreateBucket_owned_by_you, - "CreateBucket_invalid_ownership": CreateBucket_invalid_ownership, - "CreateBucket_ownership_with_acl": CreateBucket_ownership_with_acl, - "CreateBucket_as_user": CreateBucket_as_user, - "CreateDeleteBucket_success": CreateDeleteBucket_success, - "CreateBucket_default_acl": CreateBucket_default_acl, - "CreateBucket_non_default_acl": CreateBucket_non_default_acl, - "CreateBucket_default_object_lock": CreateBucket_default_object_lock, - "HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket, - "HeadBucket_success": HeadBucket_success, - "ListBuckets_as_user": ListBuckets_as_user, - "ListBuckets_as_admin": ListBuckets_as_admin, - "ListBuckets_with_prefix": ListBuckets_with_prefix, - "ListBuckets_invalid_max_buckets": ListBuckets_invalid_max_buckets, - "ListBuckets_truncated": ListBuckets_truncated, - "ListBuckets_success": ListBuckets_success, - "ListBuckets_empty_success": ListBuckets_empty_success, - "DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket, - "DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket, - "DeleteBucket_success_status_code": DeleteBucket_success_status_code, - "PutBucketOwnershipControls_non_existing_bucket": PutBucketOwnershipControls_non_existing_bucket, - "PutBucketOwnershipControls_multiple_rules": PutBucketOwnershipControls_multiple_rules, - "PutBucketOwnershipControls_invalid_ownership": PutBucketOwnershipControls_invalid_ownership, - "PutBucketOwnershipControls_success": PutBucketOwnershipControls_success, - "GetBucketOwnershipControls_non_existing_bucket": GetBucketOwnershipControls_non_existing_bucket, - "GetBucketOwnershipControls_default_ownership": GetBucketOwnershipControls_default_ownership, - "GetBucketOwnershipControls_success": GetBucketOwnershipControls_success, - "DeleteBucketOwnershipControls_non_existing_bucket": DeleteBucketOwnershipControls_non_existing_bucket, - "DeleteBucketOwnershipControls_success": DeleteBucketOwnershipControls_success, - "PutBucketTagging_non_existing_bucket": PutBucketTagging_non_existing_bucket, - "PutBucketTagging_long_tags": PutBucketTagging_long_tags, - "PutBucketTagging_success": PutBucketTagging_success, - "PutBucketTagging_success_status": PutBucketTagging_success_status, - "GetBucketTagging_non_existing_bucket": GetBucketTagging_non_existing_bucket, - "GetBucketTagging_unset_tags": GetBucketTagging_unset_tags, - "GetBucketTagging_success": GetBucketTagging_success, - "DeleteBucketTagging_non_existing_object": DeleteBucketTagging_non_existing_object, - "DeleteBucketTagging_success_status": DeleteBucketTagging_success_status, - "DeleteBucketTagging_success": DeleteBucketTagging_success, - "PutObject_non_existing_bucket": PutObject_non_existing_bucket, - "PutObject_special_chars": PutObject_special_chars, - "PutObject_invalid_long_tags": PutObject_invalid_long_tags, - "PutObject_success": PutObject_success, - "PutObject_racey_success": PutObject_racey_success, - "HeadObject_non_existing_object": HeadObject_non_existing_object, - "HeadObject_invalid_part_number": HeadObject_invalid_part_number, - "HeadObject_non_existing_mp": HeadObject_non_existing_mp, - "HeadObject_mp_success": HeadObject_mp_success, - "HeadObject_directory_object_noslash": HeadObject_directory_object_noslash, - "HeadObject_non_existing_dir_object": HeadObject_non_existing_dir_object, - "HeadObject_name_too_long": HeadObject_name_too_long, - "HeadObject_with_contenttype": HeadObject_with_contenttype, - "HeadObject_invalid_parent_dir": HeadObject_invalid_parent_dir, - "HeadObject_success": HeadObject_success, - "GetObjectAttributes_non_existing_bucket": GetObjectAttributes_non_existing_bucket, - "GetObjectAttributes_non_existing_object": GetObjectAttributes_non_existing_object, - "GetObjectAttributes_invalid_attrs": GetObjectAttributes_invalid_attrs, - "GetObjectAttributes_invalid_parent": GetObjectAttributes_invalid_parent, - "GetObjectAttributes_empty_attrs": GetObjectAttributes_empty_attrs, - "GetObjectAttributes_existing_object": GetObjectAttributes_existing_object, - "GetObject_non_existing_key": GetObject_non_existing_key, - "GetObject_directory_object_noslash": GetObject_directory_object_noslash, - "GetObject_invalid_ranges": GetObject_invalid_ranges, - "GetObject_invalid_parent": GetObject_invalid_parent, - "GetObject_with_meta": GetObject_with_meta, - "GetObject_large_object": GetObject_large_object, - "GetObject_success": GetObject_success, - "GetObject_directory_success": GetObject_directory_success, - "GetObject_by_range_success": GetObject_by_range_success, - "GetObject_by_range_resp_status": GetObject_by_range_resp_status, - "GetObject_non_existing_dir_object": GetObject_non_existing_dir_object, - "ListObjects_non_existing_bucket": ListObjects_non_existing_bucket, - "ListObjects_with_prefix": ListObjects_with_prefix, - "ListObjects_truncated": ListObjects_truncated, - "ListObjects_paginated": ListObjects_paginated, - "ListObjects_invalid_max_keys": ListObjects_invalid_max_keys, - "ListObjects_max_keys_0": ListObjects_max_keys_0, - "ListObjects_delimiter": ListObjects_delimiter, - "ListObjects_max_keys_none": ListObjects_max_keys_none, - "ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list, - "ListObjects_list_all_objs": ListObjects_list_all_objs, - "ListObjectsV2_start_after": ListObjectsV2_start_after, - "ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token, - "ListObjectsV2_start_after_not_in_list": ListObjectsV2_start_after_not_in_list, - "ListObjectsV2_start_after_empty_result": ListObjectsV2_start_after_empty_result, - "ListObjectsV2_both_delimiter_and_prefix": ListObjectsV2_both_delimiter_and_prefix, - "ListObjectsV2_single_dir_object_with_delim_and_prefix": ListObjectsV2_single_dir_object_with_delim_and_prefix, - "ListObjectsV2_truncated_common_prefixes": ListObjectsV2_truncated_common_prefixes, - "ListObjectsV2_all_objs_max_keys": ListObjectsV2_all_objs_max_keys, - "ListObjectsV2_list_all_objs": ListObjectsV2_list_all_objs, - "ListObjectsV2_invalid_parent_prefix": ListObjectsV2_invalid_parent_prefix, - "ListObjectVersions_VD_success": ListObjectVersions_VD_success, - "DeleteObject_non_existing_object": DeleteObject_non_existing_object, - "DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash, - "DeleteObject_directory_not_empty": DeleteObject_directory_not_empty, - "DeleteObject_name_too_long": DeleteObject_name_too_long, - "DeleteObject_non_existing_dir_object": DeleteObject_non_existing_dir_object, - "DeleteObject_success": DeleteObject_success, - "DeleteObject_success_status_code": DeleteObject_success_status_code, - "DeleteObjects_empty_input": DeleteObjects_empty_input, - "DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects, - "DeleteObjects_success": DeleteObjects_success, - "CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket, - "CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket, - "CopyObject_copy_to_itself": CopyObject_copy_to_itself, - "CopyObject_copy_to_itself_invalid_directive": CopyObject_copy_to_itself_invalid_directive, - "CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata, - "CopyObject_CopySource_starting_with_slash": CopyObject_CopySource_starting_with_slash, - "CopyObject_non_existing_dir_object": CopyObject_non_existing_dir_object, - "CopyObject_success": CopyObject_success, - "PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object, - "PutObjectTagging_long_tags": PutObjectTagging_long_tags, - "PutObjectTagging_success": PutObjectTagging_success, - "GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object, - "GetObjectTagging_unset_tags": GetObjectTagging_unset_tags, - "GetObjectTagging_invalid_parent": GetObjectTagging_invalid_parent, - "GetObjectTagging_success": GetObjectTagging_success, - "DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object, - "DeleteObjectTagging_success_status": DeleteObjectTagging_success_status, - "DeleteObjectTagging_success": DeleteObjectTagging_success, - "CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket, - "CreateMultipartUpload_with_metadata": CreateMultipartUpload_with_metadata, - "CreateMultipartUpload_with_invalid_tagging": CreateMultipartUpload_with_invalid_tagging, - "CreateMultipartUpload_with_tagging": CreateMultipartUpload_with_tagging, - "CreateMultipartUpload_with_content_type": CreateMultipartUpload_with_content_type, - "CreateMultipartUpload_with_object_lock": CreateMultipartUpload_with_object_lock, - "CreateMultipartUpload_with_object_lock_not_enabled": CreateMultipartUpload_with_object_lock_not_enabled, - "CreateMultipartUpload_with_object_lock_invalid_retention": CreateMultipartUpload_with_object_lock_invalid_retention, - "CreateMultipartUpload_past_retain_until_date": CreateMultipartUpload_past_retain_until_date, - "CreateMultipartUpload_success": CreateMultipartUpload_success, - "UploadPart_non_existing_bucket": UploadPart_non_existing_bucket, - "UploadPart_invalid_part_number": UploadPart_invalid_part_number, - "UploadPart_non_existing_key": UploadPart_non_existing_key, - "UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload, - "UploadPart_success": UploadPart_success, - "UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket, - "UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId, - "UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key, - "UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number, - "UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source, - "UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket, - "UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key, - "UploadPartCopy_success": UploadPartCopy_success, - "UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range, - "UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size, - "UploadPartCopy_by_range_success": UploadPartCopy_by_range_success, - "ListParts_incorrect_uploadId": ListParts_incorrect_uploadId, - "ListParts_incorrect_object_key": ListParts_incorrect_object_key, - "ListParts_invalid_max_parts": ListParts_invalid_max_parts, - "ListParts_default_max_parts": ListParts_default_max_parts, - "ListParts_truncated": ListParts_truncated, - "ListParts_success": ListParts_success, - "ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket, - "ListMultipartUploads_empty_result": ListMultipartUploads_empty_result, - "ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads, - "ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads, - "ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker, - "ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker, - "ListMultipartUploads_success": ListMultipartUploads_success, - "AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket, - "AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId, - "AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key, - "AbortMultipartUpload_success": AbortMultipartUpload_success, - "AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code, - "CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket, - "CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number, - "CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag, - "CompleteMultipartUpload_success": CompleteMultipartUpload_success, - "CompleteMultipartUpload_racey_success": CompleteMultipartUpload_racey_success, - "PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket, - "PutBucketAcl_disabled": PutBucketAcl_disabled, - "PutBucketAcl_none_of_the_options_specified": PutBucketAcl_none_of_the_options_specified, - "PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp, - "PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants, - "PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants, - "PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner, - "PutBucketAcl_invalid_owner_not_in_body": PutBucketAcl_invalid_owner_not_in_body, - "PutBucketAcl_invalid_empty_owner_id_in_body": PutBucketAcl_invalid_empty_owner_id_in_body, - "PutBucketAcl_invalid_permission_in_body": PutBucketAcl_invalid_permission_in_body, - "PutBucketAcl_invalid_grantee_type_in_body": PutBucketAcl_invalid_grantee_type_in_body, - "PutBucketAcl_empty_grantee_ID_in_body": PutBucketAcl_empty_grantee_ID_in_body, - "PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied, - "PutBucketAcl_success_grants": PutBucketAcl_success_grants, - "PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl, - "PutBucketAcl_success_acp": PutBucketAcl_success_acp, - "GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket, - "GetBucketAcl_translation_canned_public_read": GetBucketAcl_translation_canned_public_read, - "GetBucketAcl_translation_canned_public_read_write": GetBucketAcl_translation_canned_public_read_write, - "GetBucketAcl_translation_canned_private": GetBucketAcl_translation_canned_private, - "GetBucketAcl_access_denied": GetBucketAcl_access_denied, - "GetBucketAcl_success": GetBucketAcl_success, - "PutBucketPolicy_non_existing_bucket": PutBucketPolicy_non_existing_bucket, - "PutBucketPolicy_empty_statement": PutBucketPolicy_empty_statement, - "PutBucketPolicy_invalid_effect": PutBucketPolicy_invalid_effect, - "PutBucketPolicy_empty_actions_string": PutBucketPolicy_empty_actions_string, - "PutBucketPolicy_empty_actions_array": PutBucketPolicy_empty_actions_array, - "PutBucketPolicy_invalid_action": PutBucketPolicy_invalid_action, - "PutBucketPolicy_unsupported_action": PutBucketPolicy_unsupported_action, - "PutBucketPolicy_incorrect_action_wildcard_usage": PutBucketPolicy_incorrect_action_wildcard_usage, - "PutBucketPolicy_empty_principals_string": PutBucketPolicy_empty_principals_string, - "PutBucketPolicy_empty_principals_array": PutBucketPolicy_empty_principals_array, - "PutBucketPolicy_principals_aws_struct_empty_string": PutBucketPolicy_principals_aws_struct_empty_string, - "PutBucketPolicy_principals_aws_struct_empty_string_slice": PutBucketPolicy_principals_aws_struct_empty_string_slice, - "PutBucketPolicy_principals_incorrect_wildcard_usage": PutBucketPolicy_principals_incorrect_wildcard_usage, - "PutBucketPolicy_non_existing_principals": PutBucketPolicy_non_existing_principals, - "PutBucketPolicy_empty_resources_string": PutBucketPolicy_empty_resources_string, - "PutBucketPolicy_empty_resources_array": PutBucketPolicy_empty_resources_array, - "PutBucketPolicy_invalid_resource_prefix": PutBucketPolicy_invalid_resource_prefix, - "PutBucketPolicy_invalid_resource_with_starting_slash": PutBucketPolicy_invalid_resource_with_starting_slash, - "PutBucketPolicy_duplicate_resource": PutBucketPolicy_duplicate_resource, - "PutBucketPolicy_incorrect_bucket_name": PutBucketPolicy_incorrect_bucket_name, - "PutBucketPolicy_object_action_on_bucket_resource": PutBucketPolicy_object_action_on_bucket_resource, - "PutBucketPolicy_bucket_action_on_object_resource": PutBucketPolicy_bucket_action_on_object_resource, - "PutBucketPolicy_success": PutBucketPolicy_success, - "GetBucketPolicy_non_existing_bucket": GetBucketPolicy_non_existing_bucket, - "GetBucketPolicy_not_set": GetBucketPolicy_not_set, - "GetBucketPolicy_success": GetBucketPolicy_success, - "DeleteBucketPolicy_non_existing_bucket": DeleteBucketPolicy_non_existing_bucket, - "DeleteBucketPolicy_remove_before_setting": DeleteBucketPolicy_remove_before_setting, - "DeleteBucketPolicy_success": DeleteBucketPolicy_success, - "PutObjectLockConfiguration_non_existing_bucket": PutObjectLockConfiguration_non_existing_bucket, - "PutObjectLockConfiguration_empty_config": PutObjectLockConfiguration_empty_config, - "PutObjectLockConfiguration_not_enabled_on_bucket_creation": PutObjectLockConfiguration_not_enabled_on_bucket_creation, - "PutObjectLockConfiguration_invalid_status": PutObjectLockConfiguration_invalid_status, - "PutObjectLockConfiguration_invalid_mode": PutObjectLockConfiguration_invalid_mode, - "PutObjectLockConfiguration_both_years_and_days": PutObjectLockConfiguration_both_years_and_days, - "PutObjectLockConfiguration_invalid_years_days": PutObjectLockConfiguration_invalid_years_days, - "PutObjectLockConfiguration_success": PutObjectLockConfiguration_success, - "GetObjectLockConfiguration_non_existing_bucket": GetObjectLockConfiguration_non_existing_bucket, - "GetObjectLockConfiguration_unset_config": GetObjectLockConfiguration_unset_config, - "GetObjectLockConfiguration_success": GetObjectLockConfiguration_success, - "PutObjectRetention_non_existing_bucket": PutObjectRetention_non_existing_bucket, - "PutObjectRetention_non_existing_object": PutObjectRetention_non_existing_object, - "PutObjectRetention_unset_bucket_object_lock_config": PutObjectRetention_unset_bucket_object_lock_config, - "PutObjectRetention_disabled_bucket_object_lock_config": PutObjectRetention_disabled_bucket_object_lock_config, - "PutObjectRetention_expired_retain_until_date": PutObjectRetention_expired_retain_until_date, - "PutObjectRetention_invalid_mode": PutObjectRetention_invalid_mode, - "PutObjectRetention_overwrite_compliance_mode": PutObjectRetention_overwrite_compliance_mode, - "PutObjectRetention_overwrite_governance_without_bypass_specified": PutObjectRetention_overwrite_governance_without_bypass_specified, - "PutObjectRetention_overwrite_governance_with_permission": PutObjectRetention_overwrite_governance_with_permission, - "PutObjectRetention_success": PutObjectRetention_success, - "GetObjectRetention_non_existing_bucket": GetObjectRetention_non_existing_bucket, - "GetObjectRetention_non_existing_object": GetObjectRetention_non_existing_object, - "GetObjectRetention_disabled_lock": GetObjectRetention_disabled_lock, - "GetObjectRetention_unset_config": GetObjectRetention_unset_config, - "GetObjectRetention_success": GetObjectRetention_success, - "PutObjectLegalHold_non_existing_bucket": PutObjectLegalHold_non_existing_bucket, - "PutObjectLegalHold_non_existing_object": PutObjectLegalHold_non_existing_object, - "PutObjectLegalHold_invalid_body": PutObjectLegalHold_invalid_body, - "PutObjectLegalHold_invalid_status": PutObjectLegalHold_invalid_status, - "PutObjectLegalHold_unset_bucket_object_lock_config": PutObjectLegalHold_unset_bucket_object_lock_config, - "PutObjectLegalHold_disabled_bucket_object_lock_config": PutObjectLegalHold_disabled_bucket_object_lock_config, - "PutObjectLegalHold_success": PutObjectLegalHold_success, - "GetObjectLegalHold_non_existing_bucket": GetObjectLegalHold_non_existing_bucket, - "GetObjectLegalHold_non_existing_object": GetObjectLegalHold_non_existing_object, - "GetObjectLegalHold_disabled_lock": GetObjectLegalHold_disabled_lock, - "GetObjectLegalHold_unset_config": GetObjectLegalHold_unset_config, - "GetObjectLegalHold_success": GetObjectLegalHold_success, - "WORMProtection_bucket_object_lock_configuration_compliance_mode": WORMProtection_bucket_object_lock_configuration_compliance_mode, - "WORMProtection_bucket_object_lock_configuration_governance_mode": WORMProtection_bucket_object_lock_configuration_governance_mode, - "WORMProtection_bucket_object_lock_governance_bypass_delete": WORMProtection_bucket_object_lock_governance_bypass_delete, - "WORMProtection_bucket_object_lock_governance_bypass_delete_multiple": WORMProtection_bucket_object_lock_governance_bypass_delete_multiple, - "WORMProtection_object_lock_retention_compliance_locked": WORMProtection_object_lock_retention_compliance_locked, - "WORMProtection_object_lock_retention_governance_locked": WORMProtection_object_lock_retention_governance_locked, - "WORMProtection_object_lock_retention_governance_bypass_overwrite": WORMProtection_object_lock_retention_governance_bypass_overwrite, - "WORMProtection_object_lock_retention_governance_bypass_delete": WORMProtection_object_lock_retention_governance_bypass_delete, - "WORMProtection_object_lock_retention_governance_bypass_delete_mul": WORMProtection_object_lock_retention_governance_bypass_delete_mul, - "WORMProtection_object_lock_legal_hold_locked": WORMProtection_object_lock_legal_hold_locked, - "WORMProtection_root_bypass_governance_retention_delete_object": WORMProtection_root_bypass_governance_retention_delete_object, - "PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj, - "PutObject_overwrite_file_obj": PutObject_overwrite_file_obj, - "PutObject_overwrite_file_obj_with_nested_obj": PutObject_overwrite_file_obj_with_nested_obj, - "PutObject_dir_obj_with_data": PutObject_dir_obj_with_data, - "CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj, - "IAM_user_access_denied": IAM_user_access_denied, - "IAM_userplus_access_denied": IAM_userplus_access_denied, - "IAM_userplus_CreateBucket": IAM_userplus_CreateBucket, - "IAM_admin_ChangeBucketOwner": IAM_admin_ChangeBucketOwner, - "IAM_ChangeBucketOwner_back_to_root": IAM_ChangeBucketOwner_back_to_root, - "IAM_ListBuckets": IAM_ListBuckets, - "AccessControl_default_ACL_user_access_denied": AccessControl_default_ACL_user_access_denied, - "AccessControl_default_ACL_userplus_access_denied": AccessControl_default_ACL_userplus_access_denied, - "AccessControl_default_ACL_admin_successful_access": AccessControl_default_ACL_admin_successful_access, - "AccessControl_bucket_resource_single_action": AccessControl_bucket_resource_single_action, - "AccessControl_bucket_resource_all_action": AccessControl_bucket_resource_all_action, - "AccessControl_single_object_resource_actions": AccessControl_single_object_resource_actions, - "AccessControl_multi_statement_policy": AccessControl_multi_statement_policy, - "AccessControl_bucket_ownership_to_user": AccessControl_bucket_ownership_to_user, - "AccessControl_root_PutBucketAcl": AccessControl_root_PutBucketAcl, - "AccessControl_user_PutBucketAcl_with_policy_access": AccessControl_user_PutBucketAcl_with_policy_access, - "AccessControl_copy_object_with_starting_slash_for_user": AccessControl_copy_object_with_starting_slash_for_user, - "PutBucketVersioning_non_existing_bucket": PutBucketVersioning_non_existing_bucket, - "PutBucketVersioning_invalid_status": PutBucketVersioning_invalid_status, - "PutBucketVersioning_success_enabled": PutBucketVersioning_success_enabled, - "PutBucketVersioning_success_suspended": PutBucketVersioning_success_suspended, - "GetBucketVersioning_non_existing_bucket": GetBucketVersioning_non_existing_bucket, - "GetBucketVersioning_empty_response": GetBucketVersioning_empty_response, - "GetBucketVersioning_success": GetBucketVersioning_success, - "Versioning_DeleteBucket_not_empty": Versioning_DeleteBucket_not_empty, - "Versioning_PutObject_suspended_null_versionId_obj": Versioning_PutObject_suspended_null_versionId_obj, - "Versioning_PutObject_null_versionId_obj": Versioning_PutObject_null_versionId_obj, - "Versioning_PutObject_overwrite_null_versionId_obj": Versioning_PutObject_overwrite_null_versionId_obj, - "Versioning_PutObject_success": Versioning_PutObject_success, - "Versioning_CopyObject_success": Versioning_CopyObject_success, - "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, - "Versioning_CopyObject_from_an_object_version": Versioning_CopyObject_from_an_object_version, - "Versioning_CopyObject_special_chars": Versioning_CopyObject_special_chars, - "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, - "Versioning_HeadObject_invalid_parent": Versioning_HeadObject_invalid_parent, - "Versioning_HeadObject_success": Versioning_HeadObject_success, - "Versioning_HeadObject_without_versionId": Versioning_HeadObject_without_versionId, - "Versioning_HeadObject_delete_marker": Versioning_HeadObject_delete_marker, - "Versioning_GetObject_invalid_versionId": Versioning_GetObject_invalid_versionId, - "Versioning_GetObject_success": Versioning_GetObject_success, - "Versioning_GetObject_delete_marker_without_versionId": Versioning_GetObject_delete_marker_without_versionId, - "Versioning_GetObject_delete_marker": Versioning_GetObject_delete_marker, - "Versioning_GetObject_null_versionId_obj": Versioning_GetObject_null_versionId_obj, - "Versioning_GetObjectAttributes_object_version": Versioning_GetObjectAttributes_object_version, - "Versioning_GetObjectAttributes_delete_marker": Versioning_GetObjectAttributes_delete_marker, - "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, - "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, - "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, - "Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object, - "Versioning_DeleteObject_suspended": Versioning_DeleteObject_suspended, - "Versioning_DeleteObjects_success": Versioning_DeleteObjects_success, - "Versioning_DeleteObjects_delete_deleteMarkers": Versioning_DeleteObjects_delete_deleteMarkers, - "ListObjectVersions_non_existing_bucket": ListObjectVersions_non_existing_bucket, - "ListObjectVersions_list_single_object_versions": ListObjectVersions_list_single_object_versions, - "ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions, - "ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated, - "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, - "ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj, - "ListObjectVersions_single_null_versionId_object": ListObjectVersions_single_null_versionId_object, - "Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success, - "Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object, - "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, - "Versioning_UploadPartCopy_from_an_object_version": Versioning_UploadPartCopy_from_an_object_version, - "Versioning_Enable_object_lock": Versioning_Enable_object_lock, - "Versioning_status_switch_to_suspended_with_object_lock": Versioning_status_switch_to_suspended_with_object_lock, - "Versioning_PutObjectRetention_invalid_versionId": Versioning_PutObjectRetention_invalid_versionId, - "Versioning_GetObjectRetention_invalid_versionId": Versioning_GetObjectRetention_invalid_versionId, - "Versioning_Put_GetObjectRetention_success": Versioning_Put_GetObjectRetention_success, - "Versioning_PutObjectLegalHold_invalid_versionId": Versioning_PutObjectLegalHold_invalid_versionId, - "Versioning_GetObjectLegalHold_invalid_versionId": Versioning_GetObjectLegalHold_invalid_versionId, - "Versioning_Put_GetObjectLegalHold_success": Versioning_Put_GetObjectLegalHold_success, - "Versioning_WORM_obj_version_locked_with_legal_hold": Versioning_WORM_obj_version_locked_with_legal_hold, - "Versioning_WORM_obj_version_locked_with_governance_retention": Versioning_WORM_obj_version_locked_with_governance_retention, - "Versioning_WORM_obj_version_locked_with_compliance_retention": Versioning_WORM_obj_version_locked_with_compliance_retention, - "Versioning_concurrent_upload_object": Versioning_concurrent_upload_object, + "Authentication_empty_auth_header": Authentication_empty_auth_header, + "Authentication_invalid_auth_header": Authentication_invalid_auth_header, + "Authentication_unsupported_signature_version": Authentication_unsupported_signature_version, + "Authentication_malformed_credentials": Authentication_malformed_credentials, + "Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts, + "Authentication_credentials_terminated_string": Authentication_credentials_terminated_string, + "Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service, + "Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region, + "Authentication_credentials_invalid_date": Authentication_credentials_invalid_date, + "Authentication_credentials_future_date": Authentication_credentials_future_date, + "Authentication_credentials_past_date": Authentication_credentials_past_date, + "Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key, + "Authentication_invalid_signed_headers": Authentication_invalid_signed_headers, + "Authentication_missing_date_header": Authentication_missing_date_header, + "Authentication_invalid_date_header": Authentication_invalid_date_header, + "Authentication_date_mismatch": Authentication_date_mismatch, + "Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash, + "Authentication_incorrect_md5": Authentication_incorrect_md5, + "Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key, + "PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param, + "PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm, + "PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param, + "PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts, + "PresignedAuth_creds_invalid_terminator": PresignedAuth_creds_invalid_terminator, + "PresignedAuth_creds_incorrect_service": PresignedAuth_creds_incorrect_service, + "PresignedAuth_creds_incorrect_region": PresignedAuth_creds_incorrect_region, + "PresignedAuth_creds_invalid_date": PresignedAuth_creds_invalid_date, + "PresignedAuth_missing_date_query": PresignedAuth_missing_date_query, + "PresignedAuth_dates_mismatch": PresignedAuth_dates_mismatch, + "PresignedAuth_non_existing_access_key_id": PresignedAuth_non_existing_access_key_id, + "PresignedAuth_missing_signed_headers_query_param": PresignedAuth_missing_signed_headers_query_param, + "PresignedAuth_missing_expiration_query_param": PresignedAuth_missing_expiration_query_param, + "PresignedAuth_invalid_expiration_query_param": PresignedAuth_invalid_expiration_query_param, + "PresignedAuth_negative_expiration_query_param": PresignedAuth_negative_expiration_query_param, + "PresignedAuth_exceeding_expiration_query_param": PresignedAuth_exceeding_expiration_query_param, + "PresignedAuth_expired_request": PresignedAuth_expired_request, + "PresignedAuth_incorrect_secret_key": PresignedAuth_incorrect_secret_key, + "PresignedAuth_PutObject_success": PresignedAuth_PutObject_success, + "PutObject_missing_object_lock_retention_config": PutObject_missing_object_lock_retention_config, + "PutObject_name_too_long": PutObject_name_too_long, + "PutObject_with_object_lock": PutObject_with_object_lock, + "PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch, + "PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers, + "PutObject_invalid_checksum_header": PutObject_invalid_checksum_header, + "PutObject_incorrect_checksums": PutObject_incorrect_checksums, + "PutObject_checksums_success": PutObject_checksums_success, + "PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data, + "PresignedAuth_Put_GetObject_with_UTF8_chars": PresignedAuth_Put_GetObject_with_UTF8_chars, + "PresignedAuth_UploadPart": PresignedAuth_UploadPart, + "CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name, + "CreateBucket_existing_bucket": CreateBucket_existing_bucket, + "CreateBucket_owned_by_you": CreateBucket_owned_by_you, + "CreateBucket_invalid_ownership": CreateBucket_invalid_ownership, + "CreateBucket_ownership_with_acl": CreateBucket_ownership_with_acl, + "CreateBucket_as_user": CreateBucket_as_user, + "CreateDeleteBucket_success": CreateDeleteBucket_success, + "CreateBucket_default_acl": CreateBucket_default_acl, + "CreateBucket_non_default_acl": CreateBucket_non_default_acl, + "CreateBucket_default_object_lock": CreateBucket_default_object_lock, + "HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket, + "HeadBucket_success": HeadBucket_success, + "ListBuckets_as_user": ListBuckets_as_user, + "ListBuckets_as_admin": ListBuckets_as_admin, + "ListBuckets_with_prefix": ListBuckets_with_prefix, + "ListBuckets_invalid_max_buckets": ListBuckets_invalid_max_buckets, + "ListBuckets_truncated": ListBuckets_truncated, + "ListBuckets_success": ListBuckets_success, + "DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket, + "DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket, + "DeleteBucket_success_status_code": DeleteBucket_success_status_code, + "PutBucketOwnershipControls_non_existing_bucket": PutBucketOwnershipControls_non_existing_bucket, + "PutBucketOwnershipControls_multiple_rules": PutBucketOwnershipControls_multiple_rules, + "PutBucketOwnershipControls_invalid_ownership": PutBucketOwnershipControls_invalid_ownership, + "PutBucketOwnershipControls_success": PutBucketOwnershipControls_success, + "GetBucketOwnershipControls_non_existing_bucket": GetBucketOwnershipControls_non_existing_bucket, + "GetBucketOwnershipControls_default_ownership": GetBucketOwnershipControls_default_ownership, + "GetBucketOwnershipControls_success": GetBucketOwnershipControls_success, + "DeleteBucketOwnershipControls_non_existing_bucket": DeleteBucketOwnershipControls_non_existing_bucket, + "DeleteBucketOwnershipControls_success": DeleteBucketOwnershipControls_success, + "PutBucketTagging_non_existing_bucket": PutBucketTagging_non_existing_bucket, + "PutBucketTagging_long_tags": PutBucketTagging_long_tags, + "PutBucketTagging_success": PutBucketTagging_success, + "PutBucketTagging_success_status": PutBucketTagging_success_status, + "GetBucketTagging_non_existing_bucket": GetBucketTagging_non_existing_bucket, + "GetBucketTagging_unset_tags": GetBucketTagging_unset_tags, + "GetBucketTagging_success": GetBucketTagging_success, + "DeleteBucketTagging_non_existing_object": DeleteBucketTagging_non_existing_object, + "DeleteBucketTagging_success_status": DeleteBucketTagging_success_status, + "DeleteBucketTagging_success": DeleteBucketTagging_success, + "PutObject_non_existing_bucket": PutObject_non_existing_bucket, + "PutObject_special_chars": PutObject_special_chars, + "PutObject_invalid_long_tags": PutObject_invalid_long_tags, + "PutObject_success": PutObject_success, + "PutObject_racey_success": PutObject_racey_success, + "HeadObject_non_existing_object": HeadObject_non_existing_object, + "HeadObject_invalid_part_number": HeadObject_invalid_part_number, + "HeadObject_non_existing_mp": HeadObject_non_existing_mp, + "HeadObject_mp_success": HeadObject_mp_success, + "HeadObject_directory_object_noslash": HeadObject_directory_object_noslash, + "HeadObject_non_existing_dir_object": HeadObject_non_existing_dir_object, + "HeadObject_name_too_long": HeadObject_name_too_long, + "HeadObject_with_contenttype": HeadObject_with_contenttype, + "HeadObject_invalid_parent_dir": HeadObject_invalid_parent_dir, + "HeadObject_not_enabled_checksum_mode": HeadObject_not_enabled_checksum_mode, + "HeadObject_checksums": HeadObject_checksums, + "HeadObject_success": HeadObject_success, + "GetObjectAttributes_non_existing_bucket": GetObjectAttributes_non_existing_bucket, + "GetObjectAttributes_non_existing_object": GetObjectAttributes_non_existing_object, + "GetObjectAttributes_invalid_attrs": GetObjectAttributes_invalid_attrs, + "GetObjectAttributes_invalid_parent": GetObjectAttributes_invalid_parent, + "GetObjectAttributes_empty_attrs": GetObjectAttributes_empty_attrs, + "GetObjectAttributes_existing_object": GetObjectAttributes_existing_object, + "GetObjectAttributes_checksums": GetObjectAttributes_checksums, + "GetObject_non_existing_key": GetObject_non_existing_key, + "GetObject_directory_object_noslash": GetObject_directory_object_noslash, + "GetObject_invalid_ranges": GetObject_invalid_ranges, + "GetObject_invalid_parent": GetObject_invalid_parent, + "GetObject_with_meta": GetObject_with_meta, + "GetObject_large_object": GetObject_large_object, + "GetObject_checksums": GetObject_checksums, + "GetObject_success": GetObject_success, + "GetObject_directory_success": GetObject_directory_success, + "GetObject_by_range_success": GetObject_by_range_success, + "GetObject_by_range_resp_status": GetObject_by_range_resp_status, + "GetObject_non_existing_dir_object": GetObject_non_existing_dir_object, + "ListObjects_non_existing_bucket": ListObjects_non_existing_bucket, + "ListObjects_with_prefix": ListObjects_with_prefix, + "ListObjects_truncated": ListObjects_truncated, + "ListObjects_paginated": ListObjects_paginated, + "ListObjects_invalid_max_keys": ListObjects_invalid_max_keys, + "ListObjects_max_keys_0": ListObjects_max_keys_0, + "ListObjects_delimiter": ListObjects_delimiter, + "ListObjects_max_keys_none": ListObjects_max_keys_none, + "ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list, + "ListObjects_list_all_objs": ListObjects_list_all_objs, + "ListObjects_with_checksum": ListObjects_with_checksum, + "ListObjectsV2_start_after": ListObjectsV2_start_after, + "ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token, + "ListObjectsV2_start_after_not_in_list": ListObjectsV2_start_after_not_in_list, + "ListObjectsV2_start_after_empty_result": ListObjectsV2_start_after_empty_result, + "ListObjectsV2_both_delimiter_and_prefix": ListObjectsV2_both_delimiter_and_prefix, + "ListObjectsV2_single_dir_object_with_delim_and_prefix": ListObjectsV2_single_dir_object_with_delim_and_prefix, + "ListObjectsV2_truncated_common_prefixes": ListObjectsV2_truncated_common_prefixes, + "ListObjectsV2_all_objs_max_keys": ListObjectsV2_all_objs_max_keys, + "ListObjectsV2_list_all_objs": ListObjectsV2_list_all_objs, + "ListObjectsV2_with_checksum": ListObjectsV2_with_checksum, + "ListObjectVersions_VD_success": ListObjectVersions_VD_success, + "DeleteObject_non_existing_object": DeleteObject_non_existing_object, + "DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash, + "DeleteObject_name_too_long": DeleteObject_name_too_long, + "DeleteObject_non_existing_dir_object": DeleteObject_non_existing_dir_object, + "DeleteObject_success": DeleteObject_success, + "DeleteObject_success_status_code": DeleteObject_success_status_code, + "DeleteObjects_empty_input": DeleteObjects_empty_input, + "DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects, + "DeleteObjects_success": DeleteObjects_success, + "CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket, + "CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket, + "CopyObject_copy_to_itself": CopyObject_copy_to_itself, + "CopyObject_copy_to_itself_invalid_directive": CopyObject_copy_to_itself_invalid_directive, + "CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata, + "CopyObject_CopySource_starting_with_slash": CopyObject_CopySource_starting_with_slash, + "CopyObject_non_existing_dir_object": CopyObject_non_existing_dir_object, + "CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm, + "CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy, + "CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum, + "CopyObject_should_replace_the_existing_checksum": CopyObject_should_replace_the_existing_checksum, + "CopyObject_to_itself_by_replacing_the_checksum": CopyObject_to_itself_by_replacing_the_checksum, + "CopyObject_success": CopyObject_success, + "PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object, + "PutObjectTagging_long_tags": PutObjectTagging_long_tags, + "PutObjectTagging_success": PutObjectTagging_success, + "GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object, + "GetObjectTagging_unset_tags": GetObjectTagging_unset_tags, + "GetObjectTagging_invalid_parent": GetObjectTagging_invalid_parent, + "GetObjectTagging_success": GetObjectTagging_success, + "DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object, + "DeleteObjectTagging_success_status": DeleteObjectTagging_success_status, + "DeleteObjectTagging_success": DeleteObjectTagging_success, + "CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket, + "CreateMultipartUpload_with_metadata": CreateMultipartUpload_with_metadata, + "CreateMultipartUpload_with_invalid_tagging": CreateMultipartUpload_with_invalid_tagging, + "CreateMultipartUpload_with_tagging": CreateMultipartUpload_with_tagging, + "CreateMultipartUpload_with_content_type": CreateMultipartUpload_with_content_type, + "CreateMultipartUpload_with_object_lock": CreateMultipartUpload_with_object_lock, + "CreateMultipartUpload_with_object_lock_not_enabled": CreateMultipartUpload_with_object_lock_not_enabled, + "CreateMultipartUpload_with_object_lock_invalid_retention": CreateMultipartUpload_with_object_lock_invalid_retention, + "CreateMultipartUpload_past_retain_until_date": CreateMultipartUpload_past_retain_until_date, + "CreateMultipartUpload_invalid_checksum_algorithm": CreateMultipartUpload_invalid_checksum_algorithm, + "CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type": CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type, + "CreateMultipartUpload_invalid_checksum_type": CreateMultipartUpload_invalid_checksum_type, + "CreateMultipartUpload_valid_checksum_algorithm": CreateMultipartUpload_valid_checksum_algorithm, + "CreateMultipartUpload_success": CreateMultipartUpload_success, + "UploadPart_non_existing_bucket": UploadPart_non_existing_bucket, + "UploadPart_invalid_part_number": UploadPart_invalid_part_number, + "UploadPart_non_existing_key": UploadPart_non_existing_key, + "UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload, + "UploadPart_checksum_algorithm_and_header_mismatch": UploadPart_checksum_algorithm_and_header_mismatch, + "UploadPart_multiple_checksum_headers": UploadPart_multiple_checksum_headers, + "UploadPart_invalid_checksum_header": UploadPart_invalid_checksum_header, + "UploadPart_checksum_algorithm_mistmatch_on_initialization": UploadPart_checksum_algorithm_mistmatch_on_initialization, + "UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value": UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value, + "UploadPart_incorrect_checksums": UploadPart_incorrect_checksums, + "UploadPart_with_checksums_success": UploadPart_with_checksums_success, + "UploadPart_success": UploadPart_success, + "UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket, + "UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId, + "UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key, + "UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number, + "UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source, + "UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket, + "UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key, + "UploadPartCopy_success": UploadPartCopy_success, + "UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range, + "UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size, + "UploadPartCopy_by_range_success": UploadPartCopy_by_range_success, + "UploadPartCopy_should_copy_the_checksum": UploadPartCopy_should_copy_the_checksum, + "UploadPartCopy_should_not_copy_the_checksum": UploadPartCopy_should_not_copy_the_checksum, + "UploadPartCopy_should_calculate_the_checksum": UploadPartCopy_should_calculate_the_checksum, + "ListParts_incorrect_uploadId": ListParts_incorrect_uploadId, + "ListParts_incorrect_object_key": ListParts_incorrect_object_key, + "ListParts_invalid_max_parts": ListParts_invalid_max_parts, + "ListParts_default_max_parts": ListParts_default_max_parts, + "ListParts_truncated": ListParts_truncated, + "ListParts_with_checksums": ListParts_with_checksums, + "ListParts_success": ListParts_success, + "ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket, + "ListMultipartUploads_empty_result": ListMultipartUploads_empty_result, + "ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads, + "ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads, + "ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker, + "ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker, + "ListMultipartUploads_with_checksums": ListMultipartUploads_with_checksums, + "ListMultipartUploads_success": ListMultipartUploads_success, + "AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket, + "AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId, + "AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key, + "AbortMultipartUpload_success": AbortMultipartUpload_success, + "AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code, + "CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket, + "CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number, + "CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag, + "CompleteMultipartUpload_small_upload_size": CompleteMultipartUpload_small_upload_size, + "CompleteMultipartUpload_empty_parts": CompleteMultipartUpload_empty_parts, + "CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order, + "CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type, + "CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part, + "CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part, + "CompleteMultipartUpload_incorrect_checksum_part": CompleteMultipartUpload_incorrect_checksum_part, + "CompleteMultipartUpload_different_checksum_part": CompleteMultipartUpload_different_checksum_part, + "CompleteMultipartUpload_missing_part_checksum": CompleteMultipartUpload_missing_part_checksum, + "CompleteMultipartUpload_multiple_final_checksums": CompleteMultipartUpload_multiple_final_checksums, + "CompleteMultipartUpload_invalid_final_checksums": CompleteMultipartUpload_invalid_final_checksums, + "CompleteMultipartUpload_incorrect_final_checksums": CompleteMultipartUpload_incorrect_final_checksums, + "CompleteMultipartUpload_should_calculate_the_final_checksum_full_object": CompleteMultipartUpload_should_calculate_the_final_checksum_full_object, + "CompleteMultipartUpload_should_verify_the_final_checksum": CompleteMultipartUpload_should_verify_the_final_checksum, + "CompleteMultipartUpload_checksum_type_mismatch": CompleteMultipartUpload_checksum_type_mismatch, + "CompleteMultipartUpload_should_ignore_the_final_checksum": CompleteMultipartUpload_should_ignore_the_final_checksum, + "CompleteMultipartUpload_success": CompleteMultipartUpload_success, + "CompleteMultipartUpload_racey_success": CompleteMultipartUpload_racey_success, + "PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket, + "PutBucketAcl_disabled": PutBucketAcl_disabled, + "PutBucketAcl_none_of_the_options_specified": PutBucketAcl_none_of_the_options_specified, + "PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp, + "PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants, + "PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants, + "PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner, + "PutBucketAcl_invalid_owner_not_in_body": PutBucketAcl_invalid_owner_not_in_body, + "PutBucketAcl_invalid_empty_owner_id_in_body": PutBucketAcl_invalid_empty_owner_id_in_body, + "PutBucketAcl_invalid_permission_in_body": PutBucketAcl_invalid_permission_in_body, + "PutBucketAcl_invalid_grantee_type_in_body": PutBucketAcl_invalid_grantee_type_in_body, + "PutBucketAcl_empty_grantee_ID_in_body": PutBucketAcl_empty_grantee_ID_in_body, + "PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied, + "PutBucketAcl_success_grants": PutBucketAcl_success_grants, + "PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl, + "PutBucketAcl_success_acp": PutBucketAcl_success_acp, + "GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket, + "GetBucketAcl_translation_canned_public_read": GetBucketAcl_translation_canned_public_read, + "GetBucketAcl_translation_canned_public_read_write": GetBucketAcl_translation_canned_public_read_write, + "GetBucketAcl_translation_canned_private": GetBucketAcl_translation_canned_private, + "GetBucketAcl_access_denied": GetBucketAcl_access_denied, + "GetBucketAcl_success": GetBucketAcl_success, + "PutBucketPolicy_non_existing_bucket": PutBucketPolicy_non_existing_bucket, + "PutBucketPolicy_empty_statement": PutBucketPolicy_empty_statement, + "PutBucketPolicy_invalid_effect": PutBucketPolicy_invalid_effect, + "PutBucketPolicy_empty_actions_string": PutBucketPolicy_empty_actions_string, + "PutBucketPolicy_empty_actions_array": PutBucketPolicy_empty_actions_array, + "PutBucketPolicy_invalid_action": PutBucketPolicy_invalid_action, + "PutBucketPolicy_unsupported_action": PutBucketPolicy_unsupported_action, + "PutBucketPolicy_incorrect_action_wildcard_usage": PutBucketPolicy_incorrect_action_wildcard_usage, + "PutBucketPolicy_empty_principals_string": PutBucketPolicy_empty_principals_string, + "PutBucketPolicy_empty_principals_array": PutBucketPolicy_empty_principals_array, + "PutBucketPolicy_principals_aws_struct_empty_string": PutBucketPolicy_principals_aws_struct_empty_string, + "PutBucketPolicy_principals_aws_struct_empty_string_slice": PutBucketPolicy_principals_aws_struct_empty_string_slice, + "PutBucketPolicy_principals_incorrect_wildcard_usage": PutBucketPolicy_principals_incorrect_wildcard_usage, + "PutBucketPolicy_non_existing_principals": PutBucketPolicy_non_existing_principals, + "PutBucketPolicy_empty_resources_string": PutBucketPolicy_empty_resources_string, + "PutBucketPolicy_empty_resources_array": PutBucketPolicy_empty_resources_array, + "PutBucketPolicy_invalid_resource_prefix": PutBucketPolicy_invalid_resource_prefix, + "PutBucketPolicy_invalid_resource_with_starting_slash": PutBucketPolicy_invalid_resource_with_starting_slash, + "PutBucketPolicy_duplicate_resource": PutBucketPolicy_duplicate_resource, + "PutBucketPolicy_incorrect_bucket_name": PutBucketPolicy_incorrect_bucket_name, + "PutBucketPolicy_object_action_on_bucket_resource": PutBucketPolicy_object_action_on_bucket_resource, + "PutBucketPolicy_bucket_action_on_object_resource": PutBucketPolicy_bucket_action_on_object_resource, + "PutBucketPolicy_success": PutBucketPolicy_success, + "GetBucketPolicy_non_existing_bucket": GetBucketPolicy_non_existing_bucket, + "GetBucketPolicy_not_set": GetBucketPolicy_not_set, + "GetBucketPolicy_success": GetBucketPolicy_success, + "DeleteBucketPolicy_non_existing_bucket": DeleteBucketPolicy_non_existing_bucket, + "DeleteBucketPolicy_remove_before_setting": DeleteBucketPolicy_remove_before_setting, + "DeleteBucketPolicy_success": DeleteBucketPolicy_success, + "PutObjectLockConfiguration_non_existing_bucket": PutObjectLockConfiguration_non_existing_bucket, + "PutObjectLockConfiguration_empty_config": PutObjectLockConfiguration_empty_config, + "PutObjectLockConfiguration_not_enabled_on_bucket_creation": PutObjectLockConfiguration_not_enabled_on_bucket_creation, + "PutObjectLockConfiguration_invalid_status": PutObjectLockConfiguration_invalid_status, + "PutObjectLockConfiguration_invalid_mode": PutObjectLockConfiguration_invalid_mode, + "PutObjectLockConfiguration_both_years_and_days": PutObjectLockConfiguration_both_years_and_days, + "PutObjectLockConfiguration_invalid_years_days": PutObjectLockConfiguration_invalid_years_days, + "PutObjectLockConfiguration_success": PutObjectLockConfiguration_success, + "GetObjectLockConfiguration_non_existing_bucket": GetObjectLockConfiguration_non_existing_bucket, + "GetObjectLockConfiguration_unset_config": GetObjectLockConfiguration_unset_config, + "GetObjectLockConfiguration_success": GetObjectLockConfiguration_success, + "PutObjectRetention_non_existing_bucket": PutObjectRetention_non_existing_bucket, + "PutObjectRetention_non_existing_object": PutObjectRetention_non_existing_object, + "PutObjectRetention_unset_bucket_object_lock_config": PutObjectRetention_unset_bucket_object_lock_config, + "PutObjectRetention_disabled_bucket_object_lock_config": PutObjectRetention_disabled_bucket_object_lock_config, + "PutObjectRetention_expired_retain_until_date": PutObjectRetention_expired_retain_until_date, + "PutObjectRetention_invalid_mode": PutObjectRetention_invalid_mode, + "PutObjectRetention_overwrite_compliance_mode": PutObjectRetention_overwrite_compliance_mode, + "PutObjectRetention_overwrite_governance_without_bypass_specified": PutObjectRetention_overwrite_governance_without_bypass_specified, + "PutObjectRetention_overwrite_governance_with_permission": PutObjectRetention_overwrite_governance_with_permission, + "PutObjectRetention_success": PutObjectRetention_success, + "GetObjectRetention_non_existing_bucket": GetObjectRetention_non_existing_bucket, + "GetObjectRetention_non_existing_object": GetObjectRetention_non_existing_object, + "GetObjectRetention_disabled_lock": GetObjectRetention_disabled_lock, + "GetObjectRetention_unset_config": GetObjectRetention_unset_config, + "GetObjectRetention_success": GetObjectRetention_success, + "PutObjectLegalHold_non_existing_bucket": PutObjectLegalHold_non_existing_bucket, + "PutObjectLegalHold_non_existing_object": PutObjectLegalHold_non_existing_object, + "PutObjectLegalHold_invalid_body": PutObjectLegalHold_invalid_body, + "PutObjectLegalHold_invalid_status": PutObjectLegalHold_invalid_status, + "PutObjectLegalHold_unset_bucket_object_lock_config": PutObjectLegalHold_unset_bucket_object_lock_config, + "PutObjectLegalHold_disabled_bucket_object_lock_config": PutObjectLegalHold_disabled_bucket_object_lock_config, + "PutObjectLegalHold_success": PutObjectLegalHold_success, + "GetObjectLegalHold_non_existing_bucket": GetObjectLegalHold_non_existing_bucket, + "GetObjectLegalHold_non_existing_object": GetObjectLegalHold_non_existing_object, + "GetObjectLegalHold_disabled_lock": GetObjectLegalHold_disabled_lock, + "GetObjectLegalHold_unset_config": GetObjectLegalHold_unset_config, + "GetObjectLegalHold_success": GetObjectLegalHold_success, + "WORMProtection_bucket_object_lock_configuration_compliance_mode": WORMProtection_bucket_object_lock_configuration_compliance_mode, + "WORMProtection_bucket_object_lock_configuration_governance_mode": WORMProtection_bucket_object_lock_configuration_governance_mode, + "WORMProtection_bucket_object_lock_governance_bypass_delete": WORMProtection_bucket_object_lock_governance_bypass_delete, + "WORMProtection_bucket_object_lock_governance_bypass_delete_multiple": WORMProtection_bucket_object_lock_governance_bypass_delete_multiple, + "WORMProtection_object_lock_retention_compliance_locked": WORMProtection_object_lock_retention_compliance_locked, + "WORMProtection_object_lock_retention_governance_locked": WORMProtection_object_lock_retention_governance_locked, + "WORMProtection_object_lock_retention_governance_bypass_overwrite": WORMProtection_object_lock_retention_governance_bypass_overwrite, + "WORMProtection_object_lock_retention_governance_bypass_delete": WORMProtection_object_lock_retention_governance_bypass_delete, + "WORMProtection_object_lock_retention_governance_bypass_delete_mul": WORMProtection_object_lock_retention_governance_bypass_delete_mul, + "WORMProtection_object_lock_legal_hold_locked": WORMProtection_object_lock_legal_hold_locked, + "WORMProtection_root_bypass_governance_retention_delete_object": WORMProtection_root_bypass_governance_retention_delete_object, + "PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj, + "PutObject_overwrite_file_obj": PutObject_overwrite_file_obj, + "PutObject_overwrite_file_obj_with_nested_obj": PutObject_overwrite_file_obj_with_nested_obj, + "PutObject_dir_obj_with_data": PutObject_dir_obj_with_data, + "CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj, + "IAM_user_access_denied": IAM_user_access_denied, + "IAM_userplus_access_denied": IAM_userplus_access_denied, + "IAM_userplus_CreateBucket": IAM_userplus_CreateBucket, + "IAM_admin_ChangeBucketOwner": IAM_admin_ChangeBucketOwner, + "IAM_ChangeBucketOwner_back_to_root": IAM_ChangeBucketOwner_back_to_root, + "AccessControl_default_ACL_user_access_denied": AccessControl_default_ACL_user_access_denied, + "AccessControl_default_ACL_userplus_access_denied": AccessControl_default_ACL_userplus_access_denied, + "AccessControl_default_ACL_admin_successful_access": AccessControl_default_ACL_admin_successful_access, + "AccessControl_bucket_resource_single_action": AccessControl_bucket_resource_single_action, + "AccessControl_bucket_resource_all_action": AccessControl_bucket_resource_all_action, + "AccessControl_single_object_resource_actions": AccessControl_single_object_resource_actions, + "AccessControl_multi_statement_policy": AccessControl_multi_statement_policy, + "AccessControl_bucket_ownership_to_user": AccessControl_bucket_ownership_to_user, + "AccessControl_root_PutBucketAcl": AccessControl_root_PutBucketAcl, + "AccessControl_user_PutBucketAcl_with_policy_access": AccessControl_user_PutBucketAcl_with_policy_access, + "AccessControl_copy_object_with_starting_slash_for_user": AccessControl_copy_object_with_starting_slash_for_user, + "PutBucketVersioning_non_existing_bucket": PutBucketVersioning_non_existing_bucket, + "PutBucketVersioning_invalid_status": PutBucketVersioning_invalid_status, + "PutBucketVersioning_success_enabled": PutBucketVersioning_success_enabled, + "PutBucketVersioning_success_suspended": PutBucketVersioning_success_suspended, + "GetBucketVersioning_non_existing_bucket": GetBucketVersioning_non_existing_bucket, + "GetBucketVersioning_empty_response": GetBucketVersioning_empty_response, + "GetBucketVersioning_success": GetBucketVersioning_success, + "Versioning_DeleteBucket_not_empty": Versioning_DeleteBucket_not_empty, + "Versioning_PutObject_suspended_null_versionId_obj": Versioning_PutObject_suspended_null_versionId_obj, + "Versioning_PutObject_null_versionId_obj": Versioning_PutObject_null_versionId_obj, + "Versioning_PutObject_overwrite_null_versionId_obj": Versioning_PutObject_overwrite_null_versionId_obj, + "Versioning_PutObject_success": Versioning_PutObject_success, + "Versioning_CopyObject_success": Versioning_CopyObject_success, + "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, + "Versioning_CopyObject_from_an_object_version": Versioning_CopyObject_from_an_object_version, + "Versioning_CopyObject_special_chars": Versioning_CopyObject_special_chars, + "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, + "Versioning_HeadObject_invalid_parent": Versioning_HeadObject_invalid_parent, + "Versioning_HeadObject_success": Versioning_HeadObject_success, + "Versioning_HeadObject_without_versionId": Versioning_HeadObject_without_versionId, + "Versioning_HeadObject_delete_marker": Versioning_HeadObject_delete_marker, + "Versioning_GetObject_invalid_versionId": Versioning_GetObject_invalid_versionId, + "Versioning_GetObject_success": Versioning_GetObject_success, + "Versioning_GetObject_delete_marker_without_versionId": Versioning_GetObject_delete_marker_without_versionId, + "Versioning_GetObject_delete_marker": Versioning_GetObject_delete_marker, + "Versioning_GetObject_null_versionId_obj": Versioning_GetObject_null_versionId_obj, + "Versioning_GetObjectAttributes_object_version": Versioning_GetObjectAttributes_object_version, + "Versioning_GetObjectAttributes_delete_marker": Versioning_GetObjectAttributes_delete_marker, + "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, + "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, + "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, + "Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object, + "Versioning_DeleteObject_suspended": Versioning_DeleteObject_suspended, + "Versioning_DeleteObjects_success": Versioning_DeleteObjects_success, + "Versioning_DeleteObjects_delete_deleteMarkers": Versioning_DeleteObjects_delete_deleteMarkers, + "ListObjectVersions_non_existing_bucket": ListObjectVersions_non_existing_bucket, + "ListObjectVersions_list_single_object_versions": ListObjectVersions_list_single_object_versions, + "ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions, + "ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated, + "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, + "ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj, + "ListObjectVersions_single_null_versionId_object": ListObjectVersions_single_null_versionId_object, + "Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success, + "Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object, + "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, + "Versioning_UploadPartCopy_from_an_object_version": Versioning_UploadPartCopy_from_an_object_version, + "Versioning_Enable_object_lock": Versioning_Enable_object_lock, + "Versioning_status_switch_to_suspended_with_object_lock": Versioning_status_switch_to_suspended_with_object_lock, + "Versioning_PutObjectRetention_invalid_versionId": Versioning_PutObjectRetention_invalid_versionId, + "Versioning_GetObjectRetention_invalid_versionId": Versioning_GetObjectRetention_invalid_versionId, + "Versioning_Put_GetObjectRetention_success": Versioning_Put_GetObjectRetention_success, + "Versioning_PutObjectLegalHold_invalid_versionId": Versioning_PutObjectLegalHold_invalid_versionId, + "Versioning_GetObjectLegalHold_invalid_versionId": Versioning_GetObjectLegalHold_invalid_versionId, + "Versioning_Put_GetObjectLegalHold_success": Versioning_Put_GetObjectLegalHold_success, + "Versioning_WORM_obj_version_locked_with_legal_hold": Versioning_WORM_obj_version_locked_with_legal_hold, + "Versioning_WORM_obj_version_locked_with_governance_retention": Versioning_WORM_obj_version_locked_with_governance_retention, + "Versioning_WORM_obj_version_locked_with_compliance_retention": Versioning_WORM_obj_version_locked_with_compliance_retention, + "Versioning_concurrent_upload_object": Versioning_concurrent_upload_object, } } diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 08c7fb5a..da3bde44 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -2757,7 +2757,6 @@ func PutObject_special_chars(s *S3Conf) error { if err != nil { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: &bucket, @@ -2767,8 +2766,8 @@ func PutObject_special_chars(s *S3Conf) error { return err } - if !compareObjects(res.Contents, objs) { - return fmt.Errorf("expected the objects to be %v, instead got %v", + if !compareObjects(objs, res.Contents) { + return fmt.Errorf("expected the objects to be %vß, instead got %v", objStrings(objs), objStrings(res.Contents)) } @@ -2925,6 +2924,228 @@ func PutObject_with_object_lock(s *S3Conf) error { return nil } +func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error { + testName := "PutObject_checksum_algorithm_and_header_mismatch" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + ChecksumCRC32C: getPtr("m0cB1Q=="), + }) + cancel() + // FIXME: The error message for PutObject is not properly serialized by the sdk + // References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921 + + // if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + // return err + // } + if err := checkSdkApiErr(err, "InvalidRequest"); err != nil { + return err + } + + return nil + }) +} + +func PutObject_multiple_checksum_headers(s *S3Conf) error { + testName := "PutObject_multiple_checksum_headers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumSHA1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), + ChecksumCRC32C: getPtr("m0cB1Q=="), + }) + cancel() + // FIXME: The error message for PutObject is not properly serialized by the sdk + // References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921 + + // if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + // return err + // } + if err := checkSdkApiErr(err, "InvalidRequest"); err != nil { + return err + } + + return nil + }) +} + +func PutObject_invalid_checksum_header(s *S3Conf) error { + testName := "PutObject_invalid_checksum_header" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, el := range []struct { + algo string + crc32 *string + crc32c *string + sha1 *string + sha256 *string + }{ + // CRC32 tests + { + algo: "crc32", + crc32: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "crc32", + crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32 + }, + // CRC32C tests + { + algo: "crc32c", + crc32c: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "crc32c", + crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c + }, + // SHA1 tests + { + algo: "sha1", + sha1: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "sha1", + sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1 + }, + // SHA256 tests + { + algo: "sha256", + sha256: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "sha256", + sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56 + }, + } { + _, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + }, s3client) + + // FIXME: The error message for PutObject is not properly serialized by the sdk + // References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921 + + // if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + // return err + // } + if err := checkSdkApiErr(err, "InvalidRequest"); err != nil { + return err + } + } + + return nil + }) +} + +func PutObject_incorrect_checksums(s *S3Conf) error { + testName := "PutObject_incorrect_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, el := range []struct { + algo types.ChecksumAlgorithm + crc32 *string + crc32c *string + sha1 *string + sha256 *string + }{ + { + algo: types.ChecksumAlgorithmCrc32, + crc32: getPtr("DUoRhQ=="), + }, + { + algo: types.ChecksumAlgorithmCrc32c, + crc32c: getPtr("yZRlqg=="), + }, + { + algo: types.ChecksumAlgorithmSha1, + sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), + }, + { + algo: types.ChecksumAlgorithmSha256, + sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="), + }, + } { + _, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + }, s3client) + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + return err + } + } + + return nil + }) +} + +func PutObject_checksums_success(s *S3Conf) error { + testName := "PutObject_checksums_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if algo == types.ChecksumAlgorithmCrc64nvme { + continue + } + res, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: algo, + }, s3client) + if err != nil { + return err + } + + if res.res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the object checksum type to be %v, instead got %v", types.ChecksumTypeFullObject, res.res.ChecksumType) + } + + switch algo { + case types.ChecksumAlgorithmCrc32: + if res.res.ChecksumCRC32 == nil { + return fmt.Errorf("expected non empty crc32 checksum in the response") + } + case types.ChecksumAlgorithmCrc32c: + if res.res.ChecksumCRC32C == nil { + return fmt.Errorf("expected non empty crc32c checksum in the response") + } + case types.ChecksumAlgorithmSha1: + if res.res.ChecksumSHA1 == nil { + return fmt.Errorf("expected non empty sha1 checksum in the response") + } + case types.ChecksumAlgorithmSha256: + if res.res.ChecksumSHA256 == nil { + return fmt.Errorf("expected non empty sha256 checksum in the response") + } + } + } + + return nil + }) +} + func PutObject_racey_success(s *S3Conf) error { testName := "PutObject_racey_success" runF(testName) @@ -3208,6 +3429,114 @@ func HeadObject_with_contenttype(s *S3Conf) error { }) } +func HeadObject_not_enabled_checksum_mode(s *S3Conf) error { + testName := "HeadObject_not_enabled_checksum_mode" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + _, err := putObjectWithData(500, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32) + } + if res.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C) + } + if res.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1) + } + if res.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256) + } + + return nil + }) +} + +func HeadObject_checksums(s *S3Conf) error { + testName := "HeadObject_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs := []struct { + key string + checksumAlgo types.ChecksumAlgorithm + }{ + { + key: "obj-1", + checksumAlgo: types.ChecksumAlgorithmCrc32, + }, + { + key: "obj-2", + checksumAlgo: types.ChecksumAlgorithmCrc32c, + }, + { + key: "obj-3", + checksumAlgo: types.ChecksumAlgorithmSha1, + }, + { + key: "obj-4", + checksumAlgo: types.ChecksumAlgorithmSha256, + }, + } + + for i, el := range objs { + out, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumAlgorithm: el.checksumAlgo, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.ChecksumType) + } + if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { + return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32)) + } + if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C)) + } + if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { + return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1)) + } + if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { + return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256)) + } + } + + return nil + }) +} + func HeadObject_invalid_parent_dir(s *S3Conf) error { testName := "HeadObject_invalid_parent_dir" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -3467,6 +3796,77 @@ func GetObjectAttributes_existing_object(s *S3Conf) error { }) } +func GetObjectAttributes_checksums(s *S3Conf) error { + testName := "GetObjectAttributes_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs := []struct { + key string + checksumAlgo types.ChecksumAlgorithm + }{ + { + key: "obj-1", + checksumAlgo: types.ChecksumAlgorithmCrc32, + }, + { + key: "obj-2", + checksumAlgo: types.ChecksumAlgorithmCrc32c, + }, + { + key: "obj-3", + checksumAlgo: types.ChecksumAlgorithmSha1, + }, + { + key: "obj-4", + checksumAlgo: types.ChecksumAlgorithmSha256, + }, + } + + for i, el := range objs { + out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumAlgorithm: el.checksumAlgo, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &el.key, + ObjectAttributes: []types.ObjectAttributes{ + types.ObjectAttributesChecksum, + }, + }) + cancel() + if err != nil { + return err + } + + if res.Checksum == nil { + return fmt.Errorf("expected non-nil checksum in the response") + } + if res.Checksum.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType) + } + if getString(res.Checksum.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { + return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.Checksum.ChecksumCRC32)) + } + if getString(res.Checksum.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.Checksum.ChecksumCRC32C)) + } + if getString(res.Checksum.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { + return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.Checksum.ChecksumSHA1)) + } + if getString(res.Checksum.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { + return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.Checksum.ChecksumSHA256)) + } + } + return nil + }) +} + func GetObject_non_existing_key(s *S3Conf) error { testName := "GetObject_non_existing_key" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -3625,6 +4025,73 @@ func GetObject_with_meta(s *S3Conf) error { }) } +func GetObject_checksums(s *S3Conf) error { + testName := "GetObject_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs := []struct { + key string + checksumAlgo types.ChecksumAlgorithm + }{ + { + key: "obj-1", + checksumAlgo: types.ChecksumAlgorithmCrc32, + }, + { + key: "obj-2", + checksumAlgo: types.ChecksumAlgorithmCrc32c, + }, + { + key: "obj-3", + checksumAlgo: types.ChecksumAlgorithmSha1, + }, + { + key: "obj-4", + checksumAlgo: types.ChecksumAlgorithmSha256, + }, + } + + for i, el := range objs { + out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumAlgorithm: el.checksumAlgo, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &el.key, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.ChecksumType) + } + if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { + return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32)) + } + if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C)) + } + if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { + return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1)) + } + if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { + return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256)) + } + } + + return nil + }) +} + func GetObject_large_object(s *S3Conf) error { testName := "GetObject_large_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4160,6 +4627,58 @@ func ListObjects_marker_not_from_obj_list(s *S3Conf) error { }) } +func ListObjects_with_checksum(s *S3Conf) error { + testName := "ListObjects_with_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents := []types.Object{} + for i, el := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if el == types.ChecksumAlgorithmCrc64nvme { + continue + } + + key := fmt.Sprintf("obj-%v", i) + size := int64(i * 30) + out, err := putObjectWithData(size, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: el, + }, s3client) + if err != nil { + return err + } + + contents = append(contents, types.Object{ + Key: &key, + ETag: out.res.ETag, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{ + el, + }, + ChecksumType: out.res.ChecksumType, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(res.Contents, contents) { + return fmt.Errorf("expected the objects list to be %v, instead got %v", contents, res.Contents) + } + + return nil + }) +} + func ListObjects_list_all_objs(s *S3Conf) error { testName := "ListObjects_list_all_objs" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4190,7 +4709,7 @@ func ListObjects_list_all_objs(s *S3Conf) error { return fmt.Errorf("expected the Prefix to be nil, instead got %v", *out.Prefix) } - if !compareObjects(out.Contents, contents) { + if !compareObjects(contents, out.Contents) { return fmt.Errorf("expected the contents to be %v, instead got %v", contents, out.Contents) } @@ -4556,7 +5075,7 @@ func ListObjectsV2_list_all_objs(s *S3Conf) error { return fmt.Errorf("expected the Prefix to be nil, instead got %v", *out.Prefix) } - if !compareObjects(out.Contents, contents) { + if !compareObjects(contents, out.Contents) { return fmt.Errorf("expected the contents to be %v, instead got %v", contents, out.Contents) } @@ -4564,6 +5083,59 @@ func ListObjectsV2_list_all_objs(s *S3Conf) error { }) } +func ListObjectsV2_with_checksum(s *S3Conf) error { + testName := "ListObjectsV2_with_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents := []types.Object{} + + for i, el := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if el == types.ChecksumAlgorithmCrc64nvme { + continue + } + + key := fmt.Sprintf("obj-%v", i) + size := int64(i * 100) + out, err := putObjectWithData(size, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: el, + }, s3client) + if err != nil { + return err + } + + contents = append(contents, types.Object{ + Key: &key, + ETag: out.res.ETag, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{ + el, + }, + ChecksumType: out.res.ChecksumType, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(res.Contents, contents) { + return fmt.Errorf("expected the objects list to be %v, instead got %v", contents, res.Contents) + } + + return nil + }) +} + func ListObjectsV2_invalid_parent_prefix(s *S3Conf) error { testName := "ListObjectsV2_invalid_parent_prefix" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -5232,6 +5804,231 @@ func CopyObject_non_existing_dir_object(s *S3Conf) error { }) } +func CopyObject_invalid_checksum_algorithm(s *S3Conf) error { + testName := "CopyObject_invalid_checksum_algorithm" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + ChecksumAlgorithm: types.ChecksumAlgorithm("invalid_checksum_algorithm"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)); err != nil { + return err + } + + return nil + }) +} + +func CopyObject_create_checksum_on_copy(s *S3Conf) error { + testName := "CopyObject_create_checksum_on_copy" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj := "source-object" + dstObj := "destination-object" + _, err := putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }) + cancel() + if err != nil { + return err + } + + if getString(res.CopyObjectResult.ChecksumSHA256) == "" { + return fmt.Errorf("expected non nil sha256 checksum") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if getString(out.ChecksumSHA256) != getString(res.CopyObjectResult.ChecksumSHA256) { + return fmt.Errorf("expected the sha256 checksum to be %v, instead got %v", getString(res.CopyObjectResult.ChecksumSHA256), getString(out.ChecksumSHA256)) + } + + return nil + }) +} + +func CopyObject_should_copy_the_existing_checksum(s *S3Conf) error { + testName := "CopyObject_should_copy_the_existing_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj := "source-object" + dstObj := "destination-object" + out, err := putObjectWithData(100, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32c, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if res.CopyObjectResult.ChecksumCRC32C == nil { + return fmt.Errorf("expected non empty crc32c checksum") + } + if getString(res.CopyObjectResult.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.CopyObjectResult.ChecksumCRC32C)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if getString(resp.ChecksumCRC32C) != getString(res.CopyObjectResult.ChecksumCRC32C) { + return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(res.CopyObjectResult.ChecksumCRC32C), getString(resp.ChecksumCRC32C)) + } + + return nil + }) +} + +func CopyObject_should_replace_the_existing_checksum(s *S3Conf) error { + testName := "CopyObject_should_replace_the_existing_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj := "source-object" + dstObj := "destination-object" + + _, err := putObjectWithData(100, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, // replace crc32 with sha1 + }) + cancel() + if err != nil { + return err + } + + if res.CopyObjectResult.ChecksumSHA1 == nil { + return fmt.Errorf("expected non empty sha1 checksum") + } + if res.CopyObjectResult.ChecksumCRC32 != nil { + return fmt.Errorf("expected empty crc32 checksum, instead got %v", *res.CopyObjectResult.ChecksumCRC32) + } + + return nil + }) +} + +func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error { + testName := "CopyObject_to_itself_by_replacing_the_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + _, err := putObjectWithData(400, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, // replace sh256 with crc32 + MetadataDirective: types.MetadataDirectiveReplace, + }) + cancel() + if err != nil { + return err + } + + if out.CopyObjectResult.ChecksumCRC32 == nil { + return fmt.Errorf("expected non empty crc32 checksum") + } + if out.CopyObjectResult.ChecksumCRC32C != nil { + return fmt.Errorf("expected empty crc32c checksum") + } + if out.CopyObjectResult.ChecksumSHA1 != nil { + return fmt.Errorf("expected empty sha1 checksum") + } + if out.CopyObjectResult.ChecksumSHA256 != nil { + return fmt.Errorf("expected empty sha256 checksum") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumCRC32 == nil { + return fmt.Errorf("expected non empty crc32 checksum") + } + if res.ChecksumCRC32C != nil { + return fmt.Errorf("expected empty crc32c checksum") + } + if res.ChecksumSHA1 != nil { + return fmt.Errorf("expected empty sha1 checksum") + } + if res.ChecksumSHA256 != nil { + return fmt.Errorf("expected empty sha256 checksum") + } + + return nil + }) +} + func CopyObject_success(s *S3Conf) error { testName := "CopyObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -5875,6 +6672,72 @@ func CreateMultipartUpload_past_retain_until_date(s *S3Conf) error { }) } +func CreateMultipartUpload_invalid_checksum_algorithm(s *S3Conf) error { + testName := "CreateMultipartUpload_invalid_checksum_algorithm" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + ChecksumAlgorithm: types.ChecksumAlgorithm("invalid_checksum_algorithm"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)); err != nil { + return err + } + + return nil + }) +} + +func CreateMultipartUpload_invalid_checksum_type(s *S3Conf) error { + testName := "CreateMultipartUpload_invalid_checksum_type" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + _, err := createMp(s3client, bucket, "my-mp", withChecksumType(types.ChecksumType("invalid_checksum_type"))) + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")); err != nil { + return err + } + + return nil + }) +} + +func CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type(s *S3Conf) error { + testName := "CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, el := range types.ChecksumTypeComposite.Values() { + _, err := createMp(s3client, bucket, "my-mp", withChecksumType(el)) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)); err != nil { + return err + } + } + + return nil + }) +} + +func CreateMultipartUpload_valid_checksum_algorithm(s *S3Conf) error { + testName := "CreateMultipartUpload_valid_checksum_algorithm" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32c, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumAlgorithm != types.ChecksumAlgorithmCrc32c { + return fmt.Errorf("expected the checksum algorithm to be %v, instead got %v", types.ChecksumAlgorithmCrc32c, res.ChecksumAlgorithm) + } + + return nil + }) +} + func CreateMultipartUpload_with_invalid_tagging(s *S3Conf) error { testName := "CreateMultipartUpload_with_invalid_tagging" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -6009,19 +6872,26 @@ func UploadPart_non_existing_bucket(s *S3Conf) error { func UploadPart_invalid_part_number(s *S3Conf) error { testName := "UploadPart_invalid_part_number" - partNumber := int32(-10) return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: getPtr("my-obj"), - UploadId: getPtr("uploadId"), - PartNumber: &partNumber, - }) - cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + key := "my-obj" + mp, err := createMp(s3client, bucket, key) + if err != nil { return err } + for _, el := range []int32{0, -1, 10001, 2300000} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &key, + UploadId: mp.UploadId, + PartNumber: &el, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPartNumber)); err != nil { + return err + } + } + return nil }) } @@ -6045,6 +6915,318 @@ func UploadPart_non_existing_mp_upload(s *S3Conf) error { }) } +func UploadPart_checksum_algorithm_and_header_mismatch(s *S3Conf) error { + testName := "UploadPart_checksum_algorithm_and_header_mismatch" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + ChecksumCRC32C: getPtr("m0cB1Q=="), + PartNumber: &partNumber, + UploadId: mp.UploadId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_multiple_checksum_headers(s *S3Conf) error { + testName := "UploadPart_multiple_checksum_headers" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32c)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumSHA1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), + ChecksumCRC32C: getPtr("m0cB1Q=="), + UploadId: mp.UploadId, + PartNumber: &partNumber, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_invalid_checksum_header(s *S3Conf) error { + testName := "UploadPart_invalid_checksum_header" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha1)) + if err != nil { + return err + } + + partNumber := int32(1) + + for _, el := range []struct { + algo string + crc32 *string + crc32c *string + sha1 *string + sha256 *string + }{ + // CRC32 tests + { + algo: "crc32", + crc32: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "crc32", + crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32 + }, + // CRC32C tests + { + algo: "crc32c", + crc32c: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "crc32c", + crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c + }, + // SHA1 tests + { + algo: "sha1", + sha1: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "sha1", + sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1 + }, + // SHA256 tests + { + algo: "sha256", + sha256: getPtr("invalid_base64!"), // invalid base64 + }, + { + algo: "sha256", + sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56 + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + PartNumber: &partNumber, + UploadId: mp.UploadId, + }) + cancel() + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", el.algo))); err != nil { + return err + } + } + + return nil + }) +} + +func UploadPart_checksum_algorithm_mistmatch_on_initialization(s *S3Conf) error { + testName := "UploadPart_checksum_algorithm_mistmatch_on_initialization" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr(types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmSha1)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s *S3Conf) error { + testName := "UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + ChecksumSHA256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="), + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr(types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmSha256)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_incorrect_checksums(s *S3Conf) error { + testName := "UploadPart_incorrect_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for _, el := range []struct { + algo types.ChecksumAlgorithm + crc32 *string + crc32c *string + sha1 *string + sha256 *string + }{ + { + algo: types.ChecksumAlgorithmCrc32, + crc32: getPtr("DUoRhQ=="), + }, + { + algo: types.ChecksumAlgorithmCrc32c, + crc32c: getPtr("yZRlqg=="), + }, + { + algo: types.ChecksumAlgorithmSha1, + sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), + }, + { + algo: types.ChecksumAlgorithmSha256, + sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="), + }, + } { + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo)) + if err != nil { + return err + } + + body := strings.NewReader("random string body") + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + UploadId: mp.UploadId, + PartNumber: &partNumber, + Body: body, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + return err + } + } + + return nil + }) +} + +func UploadPart_with_checksums_success(s *S3Conf) error { + testName := "UploadPart_with_checksums_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if algo == types.ChecksumAlgorithmCrc64nvme { + continue + } + + mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) + if err != nil { + return err + } + + partNumber := int32(1) + data := make([]byte, i*100) + rand.Read(data) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: algo, + UploadId: mp.UploadId, + PartNumber: &partNumber, + Body: bytes.NewReader(data), + }) + cancel() + if err != nil { + return err + } + + switch algo { + case types.ChecksumAlgorithmCrc32: + if res.ChecksumCRC32 == nil { + return fmt.Errorf("expected non empty crc32 checksum in the response") + } + case types.ChecksumAlgorithmCrc32c: + if res.ChecksumCRC32C == nil { + return fmt.Errorf("expected non empty crc32c checksum in the response") + } + case types.ChecksumAlgorithmSha1: + if res.ChecksumSHA1 == nil { + return fmt.Errorf("expected non empty sha1 checksum in the response") + } + case types.ChecksumAlgorithmSha256: + if res.ChecksumSHA256 == nil { + return fmt.Errorf("expected non empty sha256 checksum in the response") + } + } + } + + return nil + }) +} + func UploadPart_non_existing_key(s *S3Conf) error { testName := "UploadPart_non_existing_key" partNumber := int32(1) @@ -6547,6 +7729,162 @@ func UploadPartCopy_by_range_success(s *S3Conf) error { }) } +func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error { + testName := "UploadPartCopy_should_copy_the_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + srcObj := "source-object" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + out, err := putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, + }, s3client) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if getString(res.CopyPartResult.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { + return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.CopyPartResult.ChecksumCRC32)) + } + if res.CopyPartResult.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32C) + } + if res.CopyPartResult.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA1) + } + if res.CopyPartResult.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA256) + } + + return nil + }) +} + +func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error { + testName := "UploadPartCopy_should_not_copy_the_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + srcObj := "source-object" + + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + _, err = putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }, s3client) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if res.CopyPartResult.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32) + } + if res.CopyPartResult.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32C) + } + if res.CopyPartResult.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA1) + } + if res.CopyPartResult.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA256) + } + + return nil + }) +} + +func UploadPartCopy_should_calculate_the_checksum(s *S3Conf) error { + testName := "UploadPartCopy_should_calculate_the_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + srcObj := "source-object" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha256)) + if err != nil { + return err + } + + _, err = putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, // different from the mp checksum (sha256) + }, s3client) + if err != nil { + return err + } + + partNumber := int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + + if res.CopyPartResult.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32) + } + if res.CopyPartResult.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.CopyPartResult.ChecksumCRC32C) + } + if res.CopyPartResult.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.CopyPartResult.ChecksumSHA1) + } + if getString(res.CopyPartResult.ChecksumSHA256) == "" { + return fmt.Errorf("expected non empty sha256 checksum") + } + + return nil + }) +} + func ListParts_incorrect_uploadId(s *S3Conf) error { testName := "ListParts_incorrect_uploadId" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -6652,7 +7990,7 @@ func ListParts_truncated(s *S3Conf) error { return err } - parts, _, err := uploadParts(s3client, 5*1024*1024, 5, bucket, obj, *out.UploadId) + parts, _, err := uploadParts(s3client, 25*1024*1024, 5, bucket, obj, *out.UploadId) if err != nil { return err } @@ -6680,7 +8018,7 @@ func ListParts_truncated(s *S3Conf) error { if *res.NextPartNumberMarker != fmt.Sprint(*parts[2].PartNumber) { return fmt.Errorf("expected next part number marker to be %v, instead got %v", fmt.Sprint(*parts[2].PartNumber), *res.NextPartNumberMarker) } - if ok := compareParts(res.Parts, parts[:3]); !ok { + if !compareParts(parts[:3], res.Parts) { return fmt.Errorf("expected the parts data to be %v, instead got %v", parts[:3], res.Parts) } @@ -6699,7 +8037,7 @@ func ListParts_truncated(s *S3Conf) error { if *res2.PartNumberMarker != *res.NextPartNumberMarker { return fmt.Errorf("expected part number marker to be %v, instead got %v", *res.NextPartNumberMarker, *res2.PartNumberMarker) } - if ok := compareParts(parts[3:], res2.Parts); !ok { + if !compareParts(parts[3:], res2.Parts) { return fmt.Errorf("expected the parts data to be %v, instead got %v", parts[3:], res2.Parts) } @@ -6707,6 +8045,48 @@ func ListParts_truncated(s *S3Conf) error { }) } +func ListParts_with_checksums(s *S3Conf) error { + testName := "ListParts_with_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if algo == types.ChecksumAlgorithmCrc64nvme { + continue + } + mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, int64((i+1)*5*1024*1024), int64(i+1), bucket, obj, *mp.UploadId, withChecksum(algo)) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + }) + cancel() + if err != nil { + return err + } + + if !compareParts(parts, res.Parts) { + return fmt.Errorf("expected the mp parts to be %v, instead got %v", parts, res.Parts) + } + } + + return nil + }) +} + func ListParts_success(s *S3Conf) error { testName := "ListParts_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -6914,6 +8294,77 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error { }) } +func ListMultipartUploads_with_checksums(s *S3Conf) error { + testName := "ListMultipartUploads_with_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + uploads := []types.MultipartUpload{} + for _, el := range []struct { + obj string + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + obj: "obj-1", + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, + }, + { + obj: "obj-2", + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + { + obj: "obj-3", + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, + }, + { + obj: "obj-4", + algo: types.ChecksumAlgorithmSha256, + t: types.ChecksumTypeComposite, + }, + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // obj: "obj-5", + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + key := el.obj + mp, err := createMp(s3client, bucket, key, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + uploads = append(uploads, types.MultipartUpload{ + Key: &key, + UploadId: mp.UploadId, + StorageClass: types.StorageClassStandard, + ChecksumAlgorithm: el.algo, + ChecksumType: el.t, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareMultipartUploads(res.Uploads, uploads) { + return fmt.Errorf("expected the final multipart uploads to be %v, instead got %v", uploads, res.Uploads) + } + + return nil + }) +} + func ListMultipartUploads_success(s *S3Conf) error { testName := "ListMultipartUploads_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -7103,8 +8554,8 @@ func CompletedMultipartUpload_non_existing_bucket(s *S3Conf) error { }) } -func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error { - testName := "CompleteMultipartUpload_invalid_part_number" +func CompleteMultipartUpload_incorrect_part_number(s *S3Conf) error { + testName := "CompleteMultipartUpload_incorrect_part_number" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" out, err := createMp(s3client, bucket, obj) @@ -7191,6 +8642,767 @@ func CompleteMultipartUpload_invalid_ETag(s *S3Conf) error { return nil }) } +func CompleteMultipartUpload_invalid_checksum_type(s *S3Conf) error { + testName := "CompleteMultipartUpload_invalid_checksum_type" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 20*1024*1024, 4, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumType("invalid_type"), + }) + cancel() + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_invalid_checksum_part(s *S3Conf) error { + testName := "CompleteMultipartUpload_invalid_checksum_part" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + }) + + if i == 0 { + cParts[0].ChecksumCRC32 = getPtr("invalid_checksum") + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeFullObject, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumPart)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_multiple_checksum_part(s *S3Conf) error { + testName := "CompleteMultipartUpload_multiple_checksum_part" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeComposite)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + }) + + if i == 0 { + cParts[0].ChecksumSHA1 = getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0=") + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeComposite, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumPart)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_incorrect_checksum_part(s *S3Conf) error { + testName := "CompleteMultipartUpload_incorrect_checksum_part" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha256), withChecksumType(types.ChecksumTypeComposite)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmSha256)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumSHA256: el.ChecksumSHA256, + }) + + if i == 0 { + cParts[0].ChecksumSHA256 = getPtr("n2alat9FhKiZXkZO18V2LLcZFM3IT8R7DjSMvK//7WU=") + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeComposite, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_different_checksum_part(s *S3Conf) error { + testName := "CompleteMultipartUpload_different_checksum_part" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32c), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32c)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32C: el.ChecksumCRC32C, + }) + + if i == 0 { + cParts[0].ChecksumSHA256 = getPtr("n2alat9FhKiZXkZO18V2LLcZFM3IT8R7DjSMvK//7WU=") + cParts[0].ChecksumCRC32C = nil + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeFullObject, + }) + cancel() + if err := checkApiErr(err, s3err.APIError{ + Code: "BadDigest", + Description: "The sha256 you specified for part 1 did not match what we received.", + HTTPStatusCode: http.StatusBadRequest, + }); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_missing_part_checksum(s *S3Conf) error { + testName := "CompleteMultipartUpload_missing_part_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha1), withChecksumType(types.ChecksumTypeComposite)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmSha1)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumSHA1: el.ChecksumSHA1, + }) + + if i == 0 { + cParts[0].ChecksumSHA1 = nil + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeComposite, + }) + cancel() + if err := checkApiErr(err, s3err.APIError{ + Code: "InvalidRequest", + Description: "The upload was created using a sha1 checksum. The complete request must include the checksum for each part. It was missing for part 1 in the request.", + HTTPStatusCode: http.StatusBadRequest, + }); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_multiple_final_checksums(s *S3Conf) error { + testName := "CompleteMultipartUpload_multiple_final_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32C, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumCRC32: getPtr("sGc9Hg=="), + ChecksumCRC32C: getPtr("/2NsFg=="), + ChecksumType: types.ChecksumTypeFullObject, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error { + testName := "CompleteMultipartUpload_invalid_final_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + for _, el := range []struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + { + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmSha256, + t: types.ChecksumTypeComposite, + }, + //FIXME: uncomment the object after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(el.algo)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32C, + ChecksumCRC32C: el.ChecksumCRC32C, + ChecksumSHA1: el.ChecksumSHA1, + ChecksumSHA256: el.ChecksumSHA256, + }) + } + + mpInput := &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: el.t, + } + + switch el.algo { + case types.ChecksumAlgorithmCrc32: + mpInput.ChecksumCRC32 = getPtr("invalid_crc32") + case types.ChecksumAlgorithmCrc32c: + mpInput.ChecksumCRC32C = getPtr("invalid_crc32c") + case types.ChecksumAlgorithmSha1: + mpInput.ChecksumSHA1 = getPtr("invalid_sha1") + case types.ChecksumAlgorithmSha256: + mpInput.ChecksumSHA256 = getPtr("invalid_sha256") + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, mpInput) + cancel() + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(el.algo))))); err != nil { + return err + } + } + + return nil + }) +} + +func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error { + testName := "CompleteMultipartUpload_incorrect_final_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + for _, el := range []struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + { + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmSha256, + t: types.ChecksumTypeComposite, + }, + //FIXME: uncomment the object after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(el.algo)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + ChecksumCRC32C: el.ChecksumCRC32C, + ChecksumSHA1: el.ChecksumSHA1, + ChecksumSHA256: el.ChecksumSHA256, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + // Provide one of the parts checksum. In any case + // the final checksum will differ from one of the parts checksum + ChecksumCRC32: cParts[0].ChecksumCRC32, + ChecksumCRC32C: cParts[0].ChecksumCRC32C, + ChecksumSHA1: cParts[0].ChecksumSHA1, + ChecksumSHA256: cParts[0].ChecksumSHA256, + ChecksumType: el.t, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + return err + } + } + + return nil + }) +} + +func CompleteMultipartUpload_should_calculate_the_final_checksum_full_object(s *S3Conf) error { + testName := "CompleteMultipartUpload_should_calculate_the_final_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + for _, el := range []struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + { + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + //FIXME: uncomment the object after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + parts, csum, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(el.algo)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + ChecksumCRC32C: el.ChecksumCRC32C, + ChecksumSHA1: el.ChecksumSHA1, + ChecksumSHA256: el.ChecksumSHA256, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: el.t, + }) + cancel() + if err != nil { + return err + } + + switch el.algo { + case types.ChecksumAlgorithmCrc32: + if getString(res.ChecksumCRC32) != csum { + return fmt.Errorf("expected the final crc32 checksum to be %v, instead got %v", csum, getString(res.ChecksumCRC32)) + } + case types.ChecksumAlgorithmCrc32c: + if getString(res.ChecksumCRC32C) != csum { + return fmt.Errorf("expected the final crc32c checksum to be %v, instead got %v", csum, getString(res.ChecksumCRC32C)) + } + } + } + + return nil + }) +} + +func CompleteMultipartUpload_should_verify_the_final_checksum(s *S3Conf) error { + testName := "CompleteMultipartUpload_should_verify_the_final_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + for _, el := range []struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + { + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + //FIXME: uncomment the object after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + parts, csum, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(el.algo)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + ChecksumCRC32C: el.ChecksumCRC32C, + ChecksumSHA1: el.ChecksumSHA1, + ChecksumSHA256: el.ChecksumSHA256, + }) + } + + mpInput := &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: el.t, + } + + switch el.algo { + case types.ChecksumAlgorithmCrc32: + mpInput.ChecksumCRC32 = &csum + case types.ChecksumAlgorithmCrc32c: + mpInput.ChecksumCRC32C = &csum + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, mpInput) + cancel() + if err != nil { + return err + } + + switch el.algo { + case types.ChecksumAlgorithmCrc32: + if getString(res.ChecksumCRC32) != csum { + return fmt.Errorf("expected the final crc32 checksum to be %v, instead got %v", csum, getString(res.ChecksumCRC32)) + } + case types.ChecksumAlgorithmCrc32c: + if getString(res.ChecksumCRC32C) != csum { + return fmt.Errorf("expected the final crc32c checksum to be %v, instead got %v", csum, getString(res.ChecksumCRC32C)) + } + } + } + + return nil + }) +} + +func CompleteMultipartUpload_checksum_type_mismatch(s *S3Conf) error { + testName := "CompleteMultipartUpload_checksum_type_mismatch" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 20*1024*1024, 4, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeComposite, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchOnMpErr(types.ChecksumTypeFullObject)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_should_ignore_the_final_checksum(s *S3Conf) error { + testName := "CompleteMultipartUpload_should_ignore_the_final_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 20*1024*1024, 4, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumSHA1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), // should ignore this + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, insted got %v", *res.ChecksumCRC32) + } + if res.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, insted got %v", *res.ChecksumCRC32C) + } + if res.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, insted got %v", *res.ChecksumSHA1) + } + if res.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, insted got %v", *res.ChecksumSHA256) + } + + return nil + }) +} func CompleteMultipartUpload_small_upload_size(s *S3Conf) error { testName := "CompleteMultipartUpload_small_upload_size" @@ -7267,6 +9479,90 @@ func CompleteMultipartUpload_empty_parts(s *S3Conf) error { }) } +func CompleteMultipartUpload_incorrect_parts_order(s *S3Conf) error { + testName := "CompleteMultipartUpload_incorrect_parts_order" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := []types.CompletedPart{} + for _, el := range parts { + compParts = append(compParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + compParts[0], compParts[1] = compParts[1], compParts[0] + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPartOrder)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error { + testName := "CompleteMultipartUpload_invalid_part_number" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + invPartNumber := int32(-4) + + compParts := []types.CompletedPart{} + for _, el := range parts { + compParts = append(compParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: &invPartNumber, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)); err != nil { + return err + } + + return nil + }) +} + func CompleteMultipartUpload_success(s *S3Conf) error { testName := "CompleteMultipartUpload_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/utils.go b/tests/integration/utils.go index c02f30c2..4a647097 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -18,11 +18,15 @@ import ( "bytes" "context" "crypto/rand" + "crypto/sha1" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/xml" "errors" "fmt" + "hash" + "hash/crc32" "io" "math/big" rnd "math/rand" @@ -475,11 +479,31 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) }, nil } -func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) { +type mpCfg struct { + checksumAlgorithm types.ChecksumAlgorithm + checksumType types.ChecksumType +} + +type mpOpt func(*mpCfg) + +func withChecksum(algo types.ChecksumAlgorithm) mpOpt { + return func(mc *mpCfg) { mc.checksumAlgorithm = algo } +} +func withChecksumType(t types.ChecksumType) mpOpt { + return func(mc *mpCfg) { mc.checksumType = t } +} + +func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) { + cfg := new(mpCfg) + for _, opt := range opts { + opt(cfg) + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &key, + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: cfg.checksumAlgorithm, + ChecksumType: cfg.checksumType, }) cancel() return out, err @@ -513,6 +537,12 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { if item.StorageClass != list2[i].StorageClass { return false } + if item.ChecksumAlgorithm != list2[i].ChecksumAlgorithm { + return false + } + if item.ChecksumType != list2[i].ChecksumType { + return false + } } return true @@ -520,16 +550,53 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { func compareParts(parts1, parts2 []types.Part) bool { if len(parts1) != len(parts2) { + fmt.Printf("list length are not equal: %v != %v\n", len(parts1), len(parts2)) return false } for i, prt := range parts1 { if *prt.PartNumber != *parts2[i].PartNumber { + fmt.Printf("partNumbers are not equal, %v != %v\n", *prt.PartNumber, *parts2[i].PartNumber) return false } if *prt.ETag != *parts2[i].ETag { + fmt.Printf("etags are not equal, %v != %v\n", *prt.ETag, *parts2[i].ETag) return false } + if *prt.Size != *parts2[i].Size { + fmt.Printf("sizes are not equal, %v != %v\n", *prt.Size, *parts2[i].Size) + return false + } + if prt.ChecksumCRC32 != nil { + if *prt.ChecksumCRC32 != getString(parts2[i].ChecksumCRC32) { + fmt.Printf("crc32 checksums are not equal, %v != %v\n", *prt.ChecksumCRC32, getString(parts2[i].ChecksumCRC32)) + return false + } + } + if prt.ChecksumCRC32C != nil { + if *prt.ChecksumCRC32C != getString(parts2[i].ChecksumCRC32C) { + fmt.Printf("crc32c checksums are not equal, %v != %v\n", *prt.ChecksumCRC32C, getString(parts2[i].ChecksumCRC32C)) + return false + } + } + if prt.ChecksumSHA1 != nil { + if *prt.ChecksumSHA1 != getString(parts2[i].ChecksumSHA1) { + fmt.Printf("sha1 checksums are not equal, %v != %v\n", *prt.ChecksumSHA1, getString(parts2[i].ChecksumSHA1)) + return false + } + } + if prt.ChecksumSHA256 != nil { + if *prt.ChecksumSHA256 != getString(parts2[i].ChecksumSHA256) { + fmt.Printf("sha256 checksums are not equal, %v != %v\n", *prt.ChecksumSHA256, getString(parts2[i].ChecksumSHA256)) + return false + } + } + if prt.ChecksumCRC64NVME != nil { + if *prt.ChecksumCRC64NVME != getString(parts2[i].ChecksumCRC64NVME) { + fmt.Printf("crc64nvme checksums are not equal, %v != %v\n", *prt.ChecksumCRC64NVME, getString(parts2[i].ChecksumCRC64NVME)) + return false + } + } } return true } @@ -647,6 +714,20 @@ func compareObjects(list1, list2 []types.Object) bool { *obj.Key, *list2[i].Key, obj.StorageClass, list2[i].StorageClass) return false } + if len(obj.ChecksumAlgorithm) != 0 { + if obj.ChecksumAlgorithm[0] != list2[i].ChecksumAlgorithm[0] { + fmt.Printf("checksum algorithms are not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, obj.ChecksumAlgorithm[0], list2[i].ChecksumAlgorithm[0]) + return false + } + } + if obj.ChecksumType != "" { + if obj.ChecksumType[0] != list2[i].ChecksumType[0] { + fmt.Printf("checksum types are not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, obj.ChecksumType[0], list2[i].ChecksumType[0]) + return false + } + } } return true @@ -711,10 +792,28 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool { return true } -func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string) (parts []types.Part, csum string, err error) { +func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) { partSize := size / partCount - hash := sha256.New() + var hash hash.Hash + + cfg := new(mpCfg) + for _, opt := range opts { + opt(cfg) + } + + switch cfg.checksumAlgorithm { + case types.ChecksumAlgorithmCrc32: + hash = crc32.NewIEEE() + case types.ChecksumAlgorithmCrc32c: + hash = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case types.ChecksumAlgorithmSha1: + hash = sha1.New() + case types.ChecksumAlgorithmSha256: + hash = sha256.New() + default: + hash = sha256.New() + } for partNumber := int64(1); partNumber <= partCount; partNumber++ { partStart := (partNumber - 1) * partSize @@ -730,26 +829,46 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) pn := int32(partNumber) out, err := client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &key, - UploadId: &uploadId, - Body: bytes.NewReader(partBuffer), - PartNumber: &pn, + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + Body: bytes.NewReader(partBuffer), + PartNumber: &pn, + ChecksumAlgorithm: cfg.checksumAlgorithm, }) cancel() if err != nil { return parts, "", err } - parts = append(parts, types.Part{ + part := types.Part{ ETag: out.ETag, PartNumber: &pn, Size: &partSize, - }) - } + } + switch cfg.checksumAlgorithm { + case types.ChecksumAlgorithmCrc32: + part.ChecksumCRC32 = out.ChecksumCRC32 + case types.ChecksumAlgorithmCrc32c: + part.ChecksumCRC32C = out.ChecksumCRC32C + case types.ChecksumAlgorithmSha1: + part.ChecksumSHA1 = out.ChecksumSHA1 + case types.ChecksumAlgorithmSha256: + part.ChecksumSHA256 = out.ChecksumSHA256 + case types.ChecksumAlgorithmCrc64nvme: + part.ChecksumCRC64NVME = out.ChecksumCRC64NVME + } + + parts = append(parts, part) + } sum := hash.Sum(nil) - csum = hex.EncodeToString(sum[:]) + + if cfg.checksumAlgorithm == "" { + csum = hex.EncodeToString(sum[:]) + } else { + csum = base64.StdEncoding.EncodeToString(sum[:]) + } return parts, csum, err }