mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-24 17:12:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1023 from versity/feat/object-integrity-checksums

Browse Source 
Implements object integrity checksums
This commit is contained in:
Ben McClelland
2025-02-19 16:12:08 -08:00
committed by GitHub
parent 714f4f7607 173518278e
commit 323600a7d3
17 changed files with 4984 additions and 638 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/azure/azure.go
+42 -20
View File
@@ -905,9 +905,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
}
// Each part is translated into an uncommitted block in a newly created blob in staging area
func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
return "", err
return nil, err
}
// TODO: request streamable version of StageBlock()
@@ -916,32 +916,34 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (eta
// the body in memory to create an io.ReadSeekCloser
rdr, err := getReadSeekCloser(input.Body)
if err != nil {
return "", err
return nil, err
}
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return "", err
return nil, err
}
// block id serves as etag here
etag = blockIDInt32ToBase64(*input.PartNumber)
etag := blockIDInt32ToBase64(*input.PartNumber)
_, err = client.StageBlock(ctx, etag, rdr, nil)
if err != nil {
return "", parseMpError(err)
return nil, parseMpError(err)
}
return etag, nil
return &s3.UploadPartOutput{
ETag: &etag,
}, nil
}
func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return s3response.CopyObjectResult{}, nil
return s3response.CopyPartResult{}, nil
}
if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
return s3response.CopyObjectResult{}, err
return s3response.CopyPartResult{}, err
}
eTag := blockIDInt32ToBase64(*input.PartNumber)
@@ -949,10 +951,10 @@ func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInp
//TODO: the action returns not implemented on azurite, maybe in production this will work?
_, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil)
if err != nil {
return s3response.CopyObjectResult{}, parseMpError(err)
return s3response.CopyPartResult{}, parseMpError(err)
}
return s3response.CopyObjectResult{}, nil
return s3response.CopyPartResult{}, nil
}
// Lists all uncommitted parts from the blob
@@ -1195,26 +1197,46 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
uncommittedBlocks := map[int32]*blockblob.Block{}
for _, el := range blockList.UncommittedBlocks {
ptNumber, err := decodeBlockId(backend.GetStringFromPtr(el.Name))
if err != nil {
return nil, fmt.Errorf("invalid block name: %w", err)
}
uncommittedBlocks[int32(ptNumber)] = el
}
slices.SortFunc(blockList.UncommittedBlocks, func(a *blockblob.Block, b *blockblob.Block) int {
ptNumber, _ := decodeBlockId(*a.Name)
nextPtNumber, _ := decodeBlockId(*b.Name)
return ptNumber - nextPtNumber
})
// The initialie values is the lower limit of partNumber: 0
var partNumber int32
last := len(blockList.UncommittedBlocks) - 1
for i, block := range blockList.UncommittedBlocks {
ptNumber, err := decodeBlockId(*block.Name)
if err != nil {
for i, part := range input.MultipartUpload.Parts {
if part.PartNumber == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
if *part.PartNumber < 1 {
return nil, s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber)
}
if *part.PartNumber <= partNumber {
return nil, s3err.GetAPIError(s3err.ErrInvalidPartOrder)
}
partNumber = *part.PartNumber
block, ok := uncommittedBlocks[*part.PartNumber]
if !ok {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
if *input.MultipartUpload.Parts[i].ETag != *block.Name {
if *part.ETag != *block.Name {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
if *input.MultipartUpload.Parts[i].PartNumber != int32(ptNumber) {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
// all parts except the last need to be greater, thena
// all parts except the last need to be greater, than
// the minimum allowed size (5 Mib)
if i < last && *block.Size < backend.MinPartSize {
return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall)
backend/backend.go
+6 -6
View File
@@ -53,8 +53,8 @@ type Backend interface {
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
// standard object operations
PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
@@ -166,11 +166,11 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
backend/posix/posix.go
+746 -95
View File
File diff suppressed because it is too large Load Diff
backend/s3proxy/s3.go
+44 -26
View File
@@ -256,8 +256,10 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
ID: *u.Owner.ID,
DisplayName: *u.Owner.DisplayName,
},
StorageClass: u.StorageClass,
Initiated: *u.Initiated,
StorageClass: u.StorageClass,
Initiated: *u.Initiated,
ChecksumAlgorithm: u.ChecksumAlgorithm,
ChecksumType: u.ChecksumType,
})
}
@@ -293,10 +295,15 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
var parts []s3response.Part
for _, p := range output.Parts {
parts = append(parts, s3response.Part{
PartNumber: int(*p.PartNumber),
LastModified: *p.LastModified,
ETag: *p.ETag,
Size: *p.Size,
PartNumber: int(*p.PartNumber),
LastModified: *p.LastModified,
ETag: *p.ETag,
Size: *p.Size,
ChecksumCRC32: p.ChecksumCRC32,
ChecksumCRC32C: p.ChecksumCRC32C,
ChecksumCRC64NVME: p.ChecksumCRC64NVME,
ChecksumSHA1: p.ChecksumSHA1,
ChecksumSHA256: p.ChecksumSHA256,
})
}
pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -329,31 +336,34 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
MaxParts: int(*output.MaxParts),
IsTruncated: *output.IsTruncated,
Parts: parts,
ChecksumAlgorithm: output.ChecksumAlgorithm,
ChecksumType: output.ChecksumType,
}, nil
}
func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
// streaming backend is not seekable,
// use unsigned payload for streaming ops
output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
))
if err != nil {
return "", handleError(err)
}
return *output.ETag, nil
return output, handleError(err)
}
func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
output, err := s.client.UploadPartCopy(ctx, input)
if err != nil {
return s3response.CopyObjectResult{}, handleError(err)
return s3response.CopyPartResult{}, handleError(err)
}
return s3response.CopyObjectResult{
LastModified: *output.CopyPartResult.LastModified,
ETag: *output.CopyPartResult.ETag,
return s3response.CopyPartResult{
LastModified: *output.CopyPartResult.LastModified,
ETag: output.CopyPartResult.ETag,
ChecksumCRC32: output.CopyPartResult.ChecksumCRC32,
ChecksumCRC32C: output.CopyPartResult.ChecksumCRC32C,
ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME,
ChecksumSHA1: output.CopyPartResult.ChecksumSHA1,
ChecksumSHA256: output.CopyPartResult.ChecksumSHA256,
}, nil
}
@@ -373,8 +383,13 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3re
}
return s3response.PutObjectOutput{
ETag: *output.ETag,
VersionID: versionID,
ETag: *output.ETag,
VersionID: versionID,
ChecksumCRC32: output.ChecksumCRC32,
ChecksumCRC32C: output.ChecksumCRC32C,
ChecksumCRC64NVME: output.ChecksumCRC64NVME,
ChecksumSHA1: output.ChecksumSHA1,
ChecksumSHA256: output.ChecksumSHA256,
}, nil
}
@@ -425,6 +440,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
ObjectSize: out.ObjectSize,
StorageClass: out.StorageClass,
ObjectParts: &parts,
Checksum: out.Checksum,
}, handleError(err)
}
@@ -837,13 +853,15 @@ func convertObjects(objs []types.Object) []s3response.Object {
for _, obj := range objs {
result = append(result, s3response.Object{
ETag: obj.ETag,
Key: obj.Key,
LastModified: obj.LastModified,
Owner: obj.Owner,
Size: obj.Size,
RestoreStatus: obj.RestoreStatus,
StorageClass: obj.StorageClass,
ETag: obj.ETag,
Key: obj.Key,
LastModified: obj.LastModified,
Owner: obj.Owner,
Size: obj.Size,
RestoreStatus: obj.RestoreStatus,
StorageClass: obj.StorageClass,
ChecksumAlgorithm: obj.ChecksumAlgorithm,
ChecksumType: obj.ChecksumType,
})
}
s3api/controllers/backend_moq_test.go
+6 -6
View File
@@ -170,10 +170,10 @@ var _ backend.Backend = &BackendMock{}
// StringFunc: func() string {
// panic("mock out the String method")
// },
// UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
// UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
// panic("mock out the UploadPart method")
// },
// UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
// UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
// panic("mock out the UploadPartCopy method")
// },
// }
@@ -331,10 +331,10 @@ type BackendMock struct {
StringFunc func() string
// UploadPartFunc mocks the UploadPart method.
UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error)
UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error)
// UploadPartCopyFunc mocks the UploadPartCopy method.
UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
// calls tracks calls to the methods.
calls struct {
@@ -2620,7 +2620,7 @@ func (mock *BackendMock) StringCalls() []struct {
}
// UploadPart calls UploadPartFunc.
func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
if mock.UploadPartFunc == nil {
panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called")
}
@@ -2656,7 +2656,7 @@ func (mock *BackendMock) UploadPartCalls() []struct {
}
// UploadPartCopy calls UploadPartCopyFunc.
func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
if mock.UploadPartCopyFunc == nil {
panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called")
}
s3api/controllers/base.go
+297 -17
View File
@@ -485,12 +485,27 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
})
}
checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode"))
if checksumMode != "" && checksumMode != types.ChecksumModeEnabled {
if c.debug {
log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode)
}
return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObject,
BucketOwner: parsedAcl.Owner,
})
}
ctx.Locals("logResBody", false)
res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
Range: &acceptRange,
VersionId: &versionId,
Bucket: &bucket,
Key: &key,
Range: &acceptRange,
VersionId: &versionId,
ChecksumMode: checksumMode,
})
if err != nil {
if res != nil {
@@ -568,6 +583,42 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Value: string(res.StorageClass),
})
}
if res.ChecksumCRC32 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: *res.ChecksumCRC32,
})
}
if res.ChecksumCRC32C != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: *res.ChecksumCRC32C,
})
}
if res.ChecksumSHA1 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: *res.ChecksumSHA1,
})
}
if res.ChecksumSHA256 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumCRC64NVME != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
}
if res.ChecksumType != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}
// Set x-amz-meta-... headers
utils.SetMetaHeaders(ctx, res.Metadata)
@@ -1962,7 +2013,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
if c.debug {
log.Printf("invalid part number: %d", partNumber)
}
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart),
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumber),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
@@ -2007,6 +2058,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
})
}
algorithm, checksums, err := utils.ParseChecksumHeaders(ctx)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionPutObject,
BucketOwner: parsedAcl.Owner,
})
}
var body io.Reader
bodyi := ctx.Locals("body-reader")
if bodyi != nil {
@@ -2016,16 +2078,59 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
ctx.Locals("logReqBody", false)
etag, err := c.be.UploadPart(ctx.Context(),
res, err := c.be.UploadPart(ctx.Context(),
&s3.UploadPartInput{
Bucket: &bucket,
Key: &keyStart,
UploadId: &uploadId,
PartNumber: &partNumber,
ContentLength: &contentLength,
Body: body,
Bucket: &bucket,
Key: &keyStart,
UploadId: &uploadId,
PartNumber: &partNumber,
ContentLength: &contentLength,
Body: body,
ChecksumAlgorithm: algorithm,
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
})
ctx.Response().Header.Set("Etag", etag)
if err == nil {
headers := []utils.CustomHeader{}
if res.ETag != nil {
headers = append(headers, utils.CustomHeader{
Key: "ETag",
Value: *res.ETag,
})
}
switch {
case res.ChecksumCRC32 != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: *res.ChecksumCRC32,
})
case res.ChecksumCRC32C != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: *res.ChecksumCRC32C,
})
case res.ChecksumCRC64NVME != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
case res.ChecksumSHA1 != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: *res.ChecksumSHA1,
})
case res.ChecksumSHA256 != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: *res.ChecksumSHA256,
})
}
utils.SetResponseHeaders(ctx, headers)
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
@@ -2256,6 +2361,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
metaDirective = types.MetadataDirectiveReplace
}
checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
err = utils.IsChecksumAlgorithmValid(checksumAlgorithm)
if err != nil {
if c.debug {
log.Printf("invalid checksum algorithm: %v", checksumAlgorithm)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCopyObject,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.CopyObject(ctx.Context(),
&s3.CopyObjectInput{
Bucket: &bucket,
@@ -2269,6 +2389,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Metadata: metadata,
MetadataDirective: metaDirective,
StorageClass: types.StorageClass(storageClass),
ChecksumAlgorithm: checksumAlgorithm,
})
if err == nil {
hdrs := []utils.CustomHeader{}
@@ -2368,6 +2489,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
})
}
algorithm, checksums, err := utils.ParseChecksumHeaders(ctx)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionPutObject,
BucketOwner: parsedAcl.Owner,
})
}
var body io.Reader
bodyi := ctx.Locals("body-reader")
if bodyi != nil {
@@ -2390,6 +2522,12 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ObjectLockRetainUntilDate: &objLock.RetainUntilDate,
ObjectLockMode: objLock.ObjectLockMode,
ObjectLockLegalHoldStatus: objLock.LegalHoldStatus,
ChecksumAlgorithm: algorithm,
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
})
if err != nil {
return SendResponse(ctx, err,
@@ -2417,6 +2555,39 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Value: res.VersionID,
})
}
switch {
case res.ChecksumCRC32 != nil:
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: *res.ChecksumCRC32,
})
case res.ChecksumCRC32C != nil:
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: *res.ChecksumCRC32C,
})
case res.ChecksumCRC64NVME != nil:
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
case res.ChecksumSHA1 != nil:
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: *res.ChecksumSHA1,
})
case res.ChecksumSHA256 != nil:
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumType != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}
utils.SetResponseHeaders(ctx, hdrs)
@@ -2934,12 +3105,27 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
})
}
checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode"))
if checksumMode != "" && checksumMode != types.ChecksumModeEnabled {
if c.debug {
log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode)
}
return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionHeadObject,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.HeadObject(ctx.Context(),
&s3.HeadObjectInput{
Bucket: &bucket,
Key: &key,
PartNumber: partNumber,
VersionId: &versionId,
Bucket: &bucket,
Key: &key,
PartNumber: partNumber,
VersionId: &versionId,
ChecksumMode: checksumMode,
})
if err != nil {
if res != nil {
@@ -3022,6 +3208,39 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
Value: string(res.StorageClass),
})
}
switch {
case res.ChecksumCRC32 != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: *res.ChecksumCRC32,
})
case res.ChecksumCRC32C != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: *res.ChecksumCRC32C,
})
case res.ChecksumCRC64NVME != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
case res.ChecksumSHA1 != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: *res.ChecksumSHA1,
})
case res.ChecksumSHA256 != nil:
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumType != "" {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}
contentType := getstring(res.ContentType)
if contentType == "" {
@@ -3232,6 +3451,35 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
})
}
_, checksums, err := utils.ParseChecksumHeaders(ctx)
if err != nil {
if c.debug {
log.Printf("err parsing checksum headers: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCompleteMultipartUpload,
BucketOwner: parsedAcl.Owner,
})
}
checksumType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
err = utils.IsChecksumTypeValid(checksumType)
if err != nil {
if c.debug {
log.Printf("invalid checksum type: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCompleteMultipartUpload,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.CompleteMultipartUpload(ctx.Context(),
&s3.CompleteMultipartUploadInput{
Bucket: &bucket,
@@ -3240,6 +3488,12 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
MultipartUpload: &types.CompletedMultipartUpload{
Parts: data.Parts,
},
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumType: checksumType,
})
if err == nil {
if getstring(res.VersionId) != "" {
@@ -3305,6 +3559,20 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
metadata := utils.GetUserMetaData(&ctx.Request().Header)
checksumAlgorithm, checksumType, err := utils.ParseCreateMpChecksumHeaders(ctx)
if err != nil {
if c.debug {
log.Printf("err parsing checksum headers: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCreateMultipartUpload,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.CreateMultipartUpload(ctx.Context(),
&s3.CreateMultipartUploadInput{
Bucket: &bucket,
@@ -3316,7 +3584,19 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
ObjectLockMode: objLockState.ObjectLockMode,
ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus,
Metadata: metadata,
ChecksumAlgorithm: checksumAlgorithm,
ChecksumType: checksumType,
})
if err == nil {
if checksumAlgorithm != "" {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "x-amz-checksum-algorithm",
Value: string(checksumAlgorithm),
},
})
}
}
return SendXMLResponse(ctx, res, err,
&MetaOpts{
Logger: c.logger,
s3api/controllers/base_test.go
+88 -4
View File
@@ -232,6 +232,9 @@ func TestS3ApiController_GetActions(t *testing.T) {
getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
tests := []struct {
name string
app *fiber.App
@@ -329,6 +332,15 @@ func TestS3ApiController_GetActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-invalid-checksum-mode",
app: app,
args: args{
req: invalidChecksumMode,
},
wantErr: false,
statusCode: 400,
},
{
name: "Get-actions-get-object-success",
app: app,
@@ -971,14 +983,14 @@ func TestS3ApiController_PutActions(t *testing.T) {
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
return s3response.PutObjectOutput{}, nil
},
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
return "hello", nil
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
return &s3.UploadPartOutput{}, nil
},
PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
return nil
},
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, nil
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
return s3response.CopyPartResult{}, nil
},
PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
return nil
@@ -1012,6 +1024,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
// CopyObject invalid checksum algorithm
cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
// PutObjectAcl success
aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
aclReq.Header.Set("X-Amz-Acl", "private")
@@ -1033,6 +1050,40 @@ func TestS3ApiController_PutActions(t *testing.T) {
invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body))
invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
// PutObject invalid checksum algorithm
invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
// PutObject invalid base64 checksum
invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64")
// PutObject invalid crc32
invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm")
// PutObject invalid crc32c
invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==")
// PutObject invalid sha1
invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==")
// PutObject invalid sha256
invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm")
// PutObject multiple checksum headers
mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=")
mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
// PutObject checksum algorithm and header mismatch
checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1")
checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
tests := []struct {
name string
app *fiber.App
@@ -1175,6 +1226,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Copy-object-invalid-checksum-algorithm",
app: app,
args: args{
req: cpyInvChecksumAlgo,
},
wantErr: false,
statusCode: 400,
},
{
name: "Copy-object-success",
app: app,
@@ -1642,6 +1702,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
})
appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil)
invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode")
tests := []struct {
name string
app *fiber.App
@@ -1658,6 +1721,15 @@ func TestS3ApiController_HeadObject(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Head-object-invalid-checksum-mode",
app: app,
args: args{
req: invChecksumMode,
},
wantErr: false,
statusCode: 400,
},
{
name: "Head-object-error",
app: appErr,
@@ -1735,6 +1807,9 @@ func TestS3ApiController_CreateActions(t *testing.T) {
})
app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil)
invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
tests := []struct {
name string
app *fiber.App
@@ -1796,6 +1871,15 @@ func TestS3ApiController_CreateActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Create-multipart-upload-invalid-checksum-algorithm",
app: app,
args: args{
req: invChecksumAlgo,
},
wantErr: false,
statusCode: 400,
},
{
name: "Create-multipart-upload-success",
app: app,
s3api/middlewares/md5.go
+1 -1
View File
@@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
}
sum := md5.Sum(ctx.Body())
calculatedSum := utils.Md5SumString(sum[:])
calculatedSum := utils.Base64SumString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
s3api/utils/auth-reader.go
+1 -1
View File
@@ -56,7 +56,7 @@ func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, de
var hr *HashReader
hashPayload := ctx.Get("X-Amz-Content-Sha256")
if !IsSpecialPayload(hashPayload) {
hr, _ = NewHashReader(r, "", HashTypeSha256)
hr, _ = NewHashReader(r, "", HashTypeSha256Hex)
} else {
hr, _ = NewHashReader(r, "", HashTypeNone)
}
s3api/utils/csum-reader.go
+135 -10
View File
@@ -16,13 +16,19 @@ package utils
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"math/bits"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
)
@@ -31,11 +37,21 @@ type HashType string
const (
// HashTypeMd5 generates MD5 checksum for the data stream
HashTypeMd5 = "md5"
// HashTypeSha256 generates SHA256 checksum for the data stream
HashTypeSha256 = "sha256"
HashTypeMd5 HashType = "md5"
// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
HashTypeSha256 HashType = "sha256"
// HashTypeSha256Hex generates SHA256 hex encoded checksum for the data stream
HashTypeSha256Hex HashType = "sha256-hex"
// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
HashTypeSha1 HashType = "sha1"
// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
HashTypeCRC32 HashType = "crc32"
// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
HashTypeCRC32C HashType = "crc32c"
// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
HashTypeCRC64NVME HashType = "crc64nvme"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone = "none"
HashTypeNone HashType = "none"
)
// HashReader is an io.Reader that calculates the checksum
@@ -62,8 +78,18 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
switch ht {
case HashTypeMd5:
hash = md5.New()
case HashTypeSha256Hex:
hash = sha256.New()
case HashTypeSha256:
hash = sha256.New()
case HashTypeSha1:
hash = sha1.New()
case HashTypeCRC32:
hash = crc32.NewIEEE()
case HashTypeCRC32C:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeCRC64NVME:
hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case HashTypeNone:
hash = noop{}
default:
@@ -88,15 +114,40 @@ func (hr *HashReader) Read(p []byte) (int, error) {
if errors.Is(readerr, io.EOF) && hr.sum != "" {
switch hr.hashType {
case HashTypeMd5:
sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
}
case HashTypeSha256:
sum := hex.EncodeToString(hr.hash.Sum(nil))
case HashTypeSha256Hex:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
}
case HashTypeCRC32:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32)
}
case HashTypeCRC32C:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32c)
}
case HashTypeSha1:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha1)
}
case HashTypeSha256:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
}
case HashTypeCRC64NVME:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
}
default:
return n, errInvalidHashType
}
@@ -104,20 +155,38 @@ func (hr *HashReader) Read(p []byte) (int, error) {
return n, readerr
}
func (hr *HashReader) SetReader(r io.Reader) {
hr.r = r
}
// Sum returns the checksum hash of the data read so far
func (hr *HashReader) Sum() string {
switch hr.hashType {
case HashTypeMd5:
return Md5SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256Hex:
return hex.EncodeToString(hr.hash.Sum(nil))
case HashTypeCRC32:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC32C:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha1:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC64NVME:
return Base64SumString(hr.hash.Sum(nil))
default:
return ""
}
}
func (hr *HashReader) Type() HashType {
return hr.hashType
}
// Md5SumString converts the hash bytes to the string checksum value
func Md5SumString(b []byte) string {
func Base64SumString(b []byte) string {
return base64.StdEncoding.EncodeToString(b)
}
@@ -128,3 +197,59 @@ func (n noop) Sum(b []byte) []byte { return []byte{} }
func (n noop) Reset() {}
func (n noop) Size() int { return 0 }
func (n noop) BlockSize() int { return 1 }
// NewCompositeChecksumReader initializes a composite checksum
// processor, which decodes and validates the provided
// checksums and returns the final checksum based on
// the previous processings.
//
// The supported checksum types are:
// - CRC32
// - CRC32C
// - SHA1
// - SHA256
func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
var hasher hash.Hash
switch ht {
case HashTypeSha256:
hasher = sha256.New()
case HashTypeSha1:
hasher = sha1.New()
case HashTypeCRC32:
hasher = crc32.NewIEEE()
case HashTypeCRC32C:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeNone:
hasher = noop{}
default:
return nil, errInvalidHashType
}
return &CompositeChecksumReader{
hasher: hasher,
}, nil
}
type CompositeChecksumReader struct {
hasher hash.Hash
}
// Decodes and writes the checksum in the hasher
func (ccr *CompositeChecksumReader) Process(checksum string) error {
data, err := base64.StdEncoding.DecodeString(checksum)
if err != nil {
return fmt.Errorf("base64 decode: %w", err)
}
_, err = ccr.hasher.Write(data)
if err != nil {
return fmt.Errorf("hash write: %w", err)
}
return nil
}
// Returns the base64 encoded composite checksum
func (ccr *CompositeChecksumReader) Sum() string {
return Base64SumString(ccr.hasher.Sum(nil))
}
s3api/utils/utils.go
+171 -1
View File
@@ -16,6 +16,7 @@ package utils
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"io"
@@ -296,7 +297,9 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp
if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
output.StorageClass = ""
}
fmt.Printf("%+v\n", output)
if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok {
output.Checksum = nil
}
return output
}
@@ -456,3 +459,170 @@ func shouldEscape(c byte) bool {
return true
}
func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) {
sdkAlgorithm := types.ChecksumAlgorithm(ctx.Get("X-Amz-Sdk-Checksum-Algorithm"))
err := IsChecksumAlgorithmValid(sdkAlgorithm)
if err != nil {
return "", nil, err
}
checksums := map[types.ChecksumAlgorithm]string{
types.ChecksumAlgorithmCrc32: ctx.Get("X-Amz-Checksum-Crc32"),
types.ChecksumAlgorithmCrc32c: ctx.Get("X-Amz-Checksum-Crc32c"),
types.ChecksumAlgorithmSha1: ctx.Get("X-Amz-Checksum-Sha1"),
types.ChecksumAlgorithmSha256: ctx.Get("X-Amz-Checksum-Sha256"),
types.ChecksumAlgorithmCrc64nvme: ctx.Get("X-Amz-Checksum-Crc64nvme"),
}
headerCtr := 0
for al, val := range checksums {
if val != "" && !IsValidChecksum(val, al) {
return sdkAlgorithm, checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al))))
}
// If any other checksum value is provided,
// rather than x-amz-sdk-checksum-algorithm
if sdkAlgorithm != "" && sdkAlgorithm != al && val != "" {
return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
}
if val != "" {
sdkAlgorithm = al
headerCtr++
}
if headerCtr > 1 {
return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
}
}
return sdkAlgorithm, checksums, nil
}
var checksumLengths = map[types.ChecksumAlgorithm]int{
types.ChecksumAlgorithmCrc32: 4,
types.ChecksumAlgorithmCrc32c: 4,
types.ChecksumAlgorithmCrc64nvme: 8,
types.ChecksumAlgorithmSha1: 20,
types.ChecksumAlgorithmSha256: 32,
}
func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
decoded, err := base64.StdEncoding.DecodeString(checksum)
if err != nil {
return false
}
expectedLength, exists := checksumLengths[algorithm]
if !exists {
return false
}
return len(decoded) == expectedLength
}
func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
if alg != "" &&
alg != types.ChecksumAlgorithmCrc32 &&
alg != types.ChecksumAlgorithmCrc32c &&
alg != types.ChecksumAlgorithmSha1 &&
alg != types.ChecksumAlgorithmSha256 &&
alg != types.ChecksumAlgorithmCrc64nvme {
return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
}
return nil
}
// Validates the provided checksum type
func IsChecksumTypeValid(t types.ChecksumType) error {
if t != "" &&
t != types.ChecksumTypeComposite &&
t != types.ChecksumTypeFullObject {
return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")
}
return nil
}
type checksumTypeSchema map[types.ChecksumType]struct{}
type checksumSchema map[types.ChecksumAlgorithm]checksumTypeSchema
// A table defining the checksum algorithm/type support
var checksumMap checksumSchema = checksumSchema{
types.ChecksumAlgorithmCrc32: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmCrc32c: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmSha1: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmSha256: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmCrc64nvme: checksumTypeSchema{
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
// Both could be empty
"": checksumTypeSchema{
"": struct{}{},
},
}
// Checks if checksum type and algorithm are supported together
func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType) error {
typeSchema := checksumMap[algo]
_, ok := typeSchema[t]
if !ok {
return s3err.GetChecksumSchemaMismatchErr(algo, t)
}
return nil
}
// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers
func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) {
algo := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
if err := IsChecksumAlgorithmValid(algo); err != nil {
return "", "", err
}
chType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
if err := IsChecksumTypeValid(chType); err != nil {
return "", "", err
}
// Verify if checksum algorithm is provided, if
// checksum type is specified
if chType != "" && algo == "" {
return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)
}
// Verify if the checksum type is supported for
// the provided checksum algorithm
if err := checkChecksumTypeAndAlgo(algo, chType); err != nil {
return algo, chType, err
}
// x-amz-checksum-type defaults to COMPOSITE
// if x-amz-checksum-algorithm is set except
// for the CRC64NVME algorithm: it defaults to FULL_OBJECT
if algo != "" && chType == "" {
if algo == types.ChecksumAlgorithmCrc64nvme {
chType = types.ChecksumTypeFullObject
} else {
chType = types.ChecksumTypeComposite
}
}
return algo, chType, nil
}
s3api/utils/utils_test.go
+322
View File
@@ -527,3 +527,325 @@ func Test_escapePath(t *testing.T) {
})
}
}
func TestIsChecksumAlgorithmValid(t *testing.T) {
type args struct {
alg types.ChecksumAlgorithm
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "empty",
args: args{
alg: "",
},
wantErr: false,
},
{
name: "crc32",
args: args{
alg: types.ChecksumAlgorithmCrc32,
},
wantErr: false,
},
{
name: "crc32c",
args: args{
alg: types.ChecksumAlgorithmCrc32c,
},
wantErr: false,
},
{
name: "sha1",
args: args{
alg: types.ChecksumAlgorithmSha1,
},
wantErr: false,
},
{
name: "sha256",
args: args{
alg: types.ChecksumAlgorithmSha256,
},
wantErr: false,
},
{
name: "crc64nvme",
args: args{
alg: types.ChecksumAlgorithmCrc64nvme,
},
wantErr: false,
},
{
name: "invalid",
args: args{
alg: types.ChecksumAlgorithm("invalid_checksum_algorithm"),
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := IsChecksumAlgorithmValid(tt.args.alg); (err != nil) != tt.wantErr {
t.Errorf("IsChecksumAlgorithmValid() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestIsValidChecksum(t *testing.T) {
type args struct {
checksum string
algorithm types.ChecksumAlgorithm
}
tests := []struct {
name string
args args
want bool
}{
{
name: "invalid-base64",
args: args{
checksum: "invalid_base64_string",
algorithm: types.ChecksumAlgorithmCrc32,
},
want: false,
},
{
name: "invalid-crc32",
args: args{
checksum: "YXNkZmFzZGZhc2Rm",
algorithm: types.ChecksumAlgorithmCrc32,
},
want: false,
},
{
name: "valid-crc32",
args: args{
checksum: "ww2FVQ==",
algorithm: types.ChecksumAlgorithmCrc32,
},
want: true,
},
{
name: "invalid-crc32c",
args: args{
checksum: "Zmdoa2doZmtnZmhr",
algorithm: types.ChecksumAlgorithmCrc32c,
},
want: false,
},
{
name: "valid-crc32c",
args: args{
checksum: "DOsb4w==",
algorithm: types.ChecksumAlgorithmCrc32c,
},
want: true,
},
{
name: "invalid-sha1",
args: args{
checksum: "YXNkZmFzZGZhc2RmYXNkZnNhZGZzYWRm",
algorithm: types.ChecksumAlgorithmSha1,
},
want: false,
},
{
name: "valid-sha1",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmSha1,
},
want: true,
},
{
name: "invalid-sha256",
args: args{
checksum: "Zmdoa2doZmtnZmhrYXNkZmFzZGZhc2RmZHNmYXNkZg==",
algorithm: types.ChecksumAlgorithmSha256,
},
want: false,
},
{
name: "valid-sha256",
args: args{
checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
algorithm: types.ChecksumAlgorithmSha256,
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsValidChecksum(tt.args.checksum, tt.args.algorithm); got != tt.want {
t.Errorf("IsValidChecksum() = %v, want %v", got, tt.want)
}
})
}
}
func TestIsChecksumTypeValid(t *testing.T) {
type args struct {
t types.ChecksumType
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "valid_FULL_OBJECT",
args: args{
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "valid_COMPOSITE",
args: args{
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "invalid",
args: args{
t: types.ChecksumType("invalid_checksum_type"),
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := IsChecksumTypeValid(tt.args.t); (err != nil) != tt.wantErr {
t.Errorf("IsChecksumTypeValid() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func Test_checkChecksumTypeAndAlgo(t *testing.T) {
type args struct {
algo types.ChecksumAlgorithm
t types.ChecksumType
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "full_object-crc32",
args: args{
algo: types.ChecksumAlgorithmCrc32,
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "full_object-crc32c",
args: args{
algo: types.ChecksumAlgorithmCrc32c,
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "full_object-sha1",
args: args{
algo: types.ChecksumAlgorithmSha1,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-sha256",
args: args{
algo: types.ChecksumAlgorithmSha1,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-crc64nvme",
args: args{
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "full_object-crc32",
args: args{
algo: types.ChecksumAlgorithmCrc32,
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "composite-crc32",
args: args{
algo: types.ChecksumAlgorithmCrc32,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-crc32c",
args: args{
algo: types.ChecksumAlgorithmCrc32c,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-sha1",
args: args{
algo: types.ChecksumAlgorithmSha1,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-sha256",
args: args{
algo: types.ChecksumAlgorithmSha256,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-crc64nvme",
args: args{
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeComposite,
},
wantErr: true,
},
{
name: "composite-empty",
args: args{
t: types.ChecksumTypeComposite,
},
wantErr: true,
},
{
name: "full_object-empty",
args: args{
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := checkChecksumTypeAndAlgo(tt.args.algo, tt.args.t); (err != nil) != tt.wantErr {
t.Errorf("checkChecksumTypeAndAlgo() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
s3err/s3err.go
+85
View File
@@ -17,7 +17,11 @@ package s3err
import (
"bytes"
"encoding/xml"
"fmt"
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
// APIError structure
@@ -74,6 +78,8 @@ const (
ErrInvalidPart
ErrEmptyParts
ErrInvalidPartNumber
ErrInvalidPartOrder
ErrInvalidCompleteMpPartNumber
ErrInternalError
ErrInvalidCopyDest
ErrInvalidCopySource
@@ -140,6 +146,10 @@ const (
ErrInvalidVersionId
ErrNoSuchVersion
ErrSuspendedVersioningNotAllowed
ErrMultipleChecksumHeaders
ErrInvalidChecksumAlgorithm
ErrInvalidChecksumPart
ErrChecksumTypeWithAlgo
// Non-AWS errors
ErrExistingObjectIsDirectory
@@ -264,6 +274,16 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Part number must be an integer between 1 and 10000, inclusive.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidPartOrder: {
Code: "InvalidPartOrder",
Description: "The list of parts was not in ascending order. Parts must be ordered by part number.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidCompleteMpPartNumber: {
Code: "InvalidArgument",
Description: "PartNumber must be >= 1",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidCopyDest: {
Code: "InvalidRequest",
Description: "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -584,6 +604,26 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMultipleChecksumHeaders: {
Code: "InvalidRequest",
Description: "Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidChecksumAlgorithm: {
Code: "InvalidRequest",
Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidChecksumPart: {
Code: "InvalidArgument",
Description: "Invalid Base64 or multiple checksums present in request",
HTTPStatusCode: http.StatusBadRequest,
},
ErrChecksumTypeWithAlgo: {
Code: "InvalidRequest",
Description: "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.",
HTTPStatusCode: http.StatusBadRequest,
},
// non aws errors
ErrExistingObjectIsDirectory: {
@@ -678,3 +718,48 @@ func encodeResponse(response interface{}) []byte {
e.Encode(response)
return bytesBuffer.Bytes()
}
// Returns invalid checksum error with the provided header in the error description
func GetInvalidChecksumHeaderErr(header string) APIError {
return APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("Value for %v header is invalid.", header),
HTTPStatusCode: http.StatusBadRequest,
}
}
// Returns checksum type mismatch APIError
func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIError {
return APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("Checksum Type mismatch occurred, expected checksum Type: %v, actual checksum Type: %v", expected, actual),
HTTPStatusCode: http.StatusBadRequest,
}
}
// Returns incorrect checksum APIError
func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError {
return APIError{
Code: "BadDigest",
Description: fmt.Sprintf("The %v you specified did not match the calculated checksum.", strings.ToLower(string(algo))),
HTTPStatusCode: http.StatusBadRequest,
}
}
// Returns checksum type mismatch error with checksum algorithm
func GetChecksumSchemaMismatchErr(algo types.ChecksumAlgorithm, t types.ChecksumType) APIError {
return APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("The %v checksum type cannot be used with the %v checksum algorithm.", algo, strings.ToLower(string(t))),
HTTPStatusCode: http.StatusBadRequest,
}
}
// Returns checksum type mismatch error for multipart uploads
func GetChecksumTypeMismatchOnMpErr(t types.ChecksumType) APIError {
return APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("The upload was created using the %v checksum mode. The complete request must use the same checksum mode.", t),
HTTPStatusCode: http.StatusBadRequest,
}
}
s3response/s3response.go
+65 -22
View File
@@ -29,16 +29,27 @@ const (
)
type PutObjectOutput struct {
ETag string
VersionID string
ETag string
VersionID string
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumType types.ChecksumType
}
// Part describes part metadata.
type Part struct {
PartNumber int
LastModified time.Time
ETag string
Size int64
PartNumber int
LastModified time.Time
ETag string
Size int64
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
}
func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -59,9 +70,11 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
Bucket string
Key string
UploadID string `xml:"UploadId"`
Bucket string
Key string
UploadID string `xml:"UploadId"`
ChecksumAlgorithm types.ChecksumAlgorithm
ChecksumType types.ChecksumType
Initiator Initiator
Owner Owner
@@ -101,6 +114,7 @@ type GetObjectAttributesResponse struct {
ObjectSize *int64
StorageClass types.StorageClass `xml:",omitempty"`
ObjectParts *ObjectParts
Checksum *types.Checksum
// Not included in the response body
VersionId *string
@@ -169,13 +183,15 @@ type ListObjectsV2Result struct {
}
type Object struct {
ETag *string
Key *string
LastModified *time.Time
Owner *types.Owner
RestoreStatus *types.RestoreStatus
Size *int64
StorageClass types.ObjectStorageClass
ChecksumAlgorithm []types.ChecksumAlgorithm
ChecksumType types.ChecksumType
ETag *string
Key *string
LastModified *time.Time
Owner *types.Owner
RestoreStatus *types.RestoreStatus
Size *int64
StorageClass types.ObjectStorageClass
}
func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -197,12 +213,14 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
// Upload describes in progress multipart upload
type Upload struct {
Key string
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
StorageClass types.StorageClass
Initiated time.Time
Key string
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
StorageClass types.StorageClass
Initiated time.Time
ChecksumAlgorithm types.ChecksumAlgorithm
ChecksumType types.ChecksumType
}
func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -332,6 +350,20 @@ type CopyObjectResult struct {
CopySourceVersionId string `xml:"-"`
}
type CopyPartResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
LastModified time.Time
ETag *string
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
// not included in the body
CopySourceVersionId string `xml:"-"`
}
func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type Alias CopyObjectResult
aux := &struct {
@@ -467,3 +499,14 @@ func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
type ListBucketsResult struct {
Buckets []Bucket
}
type Checksum struct {
Algorithm types.ChecksumAlgorithm
Type types.ChecksumType
CRC32 *string
CRC32C *string
SHA1 *string
SHA256 *string
CRC64NVME *string
}
tests/integration/group-tests.go
+526 -395
View File
File diff suppressed because it is too large Load Diff
tests/integration/tests.go
+2316 -20
View File
File diff suppressed because it is too large Load Diff
tests/integration/utils.go
+133 -14
View File
@@ -18,11 +18,15 @@ import (
"bytes"
"context"
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"errors"
"fmt"
"hash"
"hash/crc32"
"io"
"math/big"
rnd "math/rand"
@@ -475,11 +479,31 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client)
}, nil
}
func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
type mpCfg struct {
checksumAlgorithm types.ChecksumAlgorithm
checksumType types.ChecksumType
}
type mpOpt func(*mpCfg)
func withChecksum(algo types.ChecksumAlgorithm) mpOpt {
return func(mc *mpCfg) { mc.checksumAlgorithm = algo }
}
func withChecksumType(t types.ChecksumType) mpOpt {
return func(mc *mpCfg) { mc.checksumType = t }
}
func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) {
cfg := new(mpCfg)
for _, opt := range opts {
opt(cfg)
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &key,
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: cfg.checksumAlgorithm,
ChecksumType: cfg.checksumType,
})
cancel()
return out, err
@@ -513,6 +537,12 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
if item.StorageClass != list2[i].StorageClass {
return false
}
if item.ChecksumAlgorithm != list2[i].ChecksumAlgorithm {
return false
}
if item.ChecksumType != list2[i].ChecksumType {
return false
}
}
return true
@@ -520,16 +550,53 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
func compareParts(parts1, parts2 []types.Part) bool {
if len(parts1) != len(parts2) {
fmt.Printf("list length are not equal: %v != %v\n", len(parts1), len(parts2))
return false
}
for i, prt := range parts1 {
if *prt.PartNumber != *parts2[i].PartNumber {
fmt.Printf("partNumbers are not equal, %v != %v\n", *prt.PartNumber, *parts2[i].PartNumber)
return false
}
if *prt.ETag != *parts2[i].ETag {
fmt.Printf("etags are not equal, %v != %v\n", *prt.ETag, *parts2[i].ETag)
return false
}
if *prt.Size != *parts2[i].Size {
fmt.Printf("sizes are not equal, %v != %v\n", *prt.Size, *parts2[i].Size)
return false
}
if prt.ChecksumCRC32 != nil {
if *prt.ChecksumCRC32 != getString(parts2[i].ChecksumCRC32) {
fmt.Printf("crc32 checksums are not equal, %v != %v\n", *prt.ChecksumCRC32, getString(parts2[i].ChecksumCRC32))
return false
}
}
if prt.ChecksumCRC32C != nil {
if *prt.ChecksumCRC32C != getString(parts2[i].ChecksumCRC32C) {
fmt.Printf("crc32c checksums are not equal, %v != %v\n", *prt.ChecksumCRC32C, getString(parts2[i].ChecksumCRC32C))
return false
}
}
if prt.ChecksumSHA1 != nil {
if *prt.ChecksumSHA1 != getString(parts2[i].ChecksumSHA1) {
fmt.Printf("sha1 checksums are not equal, %v != %v\n", *prt.ChecksumSHA1, getString(parts2[i].ChecksumSHA1))
return false
}
}
if prt.ChecksumSHA256 != nil {
if *prt.ChecksumSHA256 != getString(parts2[i].ChecksumSHA256) {
fmt.Printf("sha256 checksums are not equal, %v != %v\n", *prt.ChecksumSHA256, getString(parts2[i].ChecksumSHA256))
return false
}
}
if prt.ChecksumCRC64NVME != nil {
if *prt.ChecksumCRC64NVME != getString(parts2[i].ChecksumCRC64NVME) {
fmt.Printf("crc64nvme checksums are not equal, %v != %v\n", *prt.ChecksumCRC64NVME, getString(parts2[i].ChecksumCRC64NVME))
return false
}
}
}
return true
}
@@ -647,6 +714,20 @@ func compareObjects(list1, list2 []types.Object) bool {
*obj.Key, *list2[i].Key, obj.StorageClass, list2[i].StorageClass)
return false
}
if len(obj.ChecksumAlgorithm) != 0 {
if obj.ChecksumAlgorithm[0] != list2[i].ChecksumAlgorithm[0] {
fmt.Printf("checksum algorithms are not equal: (%q %q) %v != %v\n",
*obj.Key, *list2[i].Key, obj.ChecksumAlgorithm[0], list2[i].ChecksumAlgorithm[0])
return false
}
}
if obj.ChecksumType != "" {
if obj.ChecksumType[0] != list2[i].ChecksumType[0] {
fmt.Printf("checksum types are not equal: (%q %q) %v != %v\n",
*obj.Key, *list2[i].Key, obj.ChecksumType[0], list2[i].ChecksumType[0])
return false
}
}
}
return true
@@ -711,10 +792,28 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool {
return true
}
func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string) (parts []types.Part, csum string, err error) {
func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) {
partSize := size / partCount
hash := sha256.New()
var hash hash.Hash
cfg := new(mpCfg)
for _, opt := range opts {
opt(cfg)
}
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
hash = crc32.NewIEEE()
case types.ChecksumAlgorithmCrc32c:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case types.ChecksumAlgorithmSha1:
hash = sha1.New()
case types.ChecksumAlgorithmSha256:
hash = sha256.New()
default:
hash = sha256.New()
}
for partNumber := int64(1); partNumber <= partCount; partNumber++ {
partStart := (partNumber - 1) * partSize
@@ -730,26 +829,46 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
pn := int32(partNumber)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
ChecksumAlgorithm: cfg.checksumAlgorithm,
})
cancel()
if err != nil {
return parts, "", err
}
parts = append(parts, types.Part{
part := types.Part{
ETag: out.ETag,
PartNumber: &pn,
Size: &partSize,
})
}
}
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
part.ChecksumCRC32 = out.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
part.ChecksumCRC32C = out.ChecksumCRC32C
case types.ChecksumAlgorithmSha1:
part.ChecksumSHA1 = out.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
part.ChecksumSHA256 = out.ChecksumSHA256
case types.ChecksumAlgorithmCrc64nvme:
part.ChecksumCRC64NVME = out.ChecksumCRC64NVME
}
parts = append(parts, part)
}
sum := hash.Sum(nil)
csum = hex.EncodeToString(sum[:])
if cfg.checksumAlgorithm == "" {
csum = hex.EncodeToString(sum[:])
} else {
csum = base64.StdEncoding.EncodeToString(sum[:])
}
return parts, csum, err
}