mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-04 11:03:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Adds the CRC64NVME checksum support in the gateway. Adds checksum-type support for the checksum implementation

Browse Source
This commit is contained in:
niksis02
2025-02-16 17:10:06 +04:00
parent 6956757557
commit 132d0ae631
11 changed files with 2220 additions and 847 deletions
Download Patch File Download Diff File Expand all files Collapse all files

494
backend/posix/posix.go
View File

@@ -1314,14 +1314,13 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa
// Set object checksum algorithm
if mpu.ChecksumAlgorithm != "" {
err := p.storeChecksums(nil, bucket, filepath.Join(objdir, uploadID), s3response.Checksum{
Algorithms: []types.ChecksumAlgorithm{
mpu.ChecksumAlgorithm,
},
Algorithm: mpu.ChecksumAlgorithm,
Type: mpu.ChecksumType,
})
if err != nil {
// cleanup object if returning error
os.RemoveAll(filepath.Join(tmppath, uploadID))
os.Remove(tmppath)
_ = os.RemoveAll(filepath.Join(tmppath, uploadID))
_ = os.Remove(tmppath)
return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("store mp checksum algorithm: %w", err)
}
}
@@ -1352,6 +1351,21 @@ func (p *Posix) getChownIDs(acct auth.Account) (int, int, bool) {
return uid, gid, needsChown
}
func getPartChecksum(algo types.ChecksumAlgorithm, part types.CompletedPart) string {
switch algo {
case types.ChecksumAlgorithmCrc32:
return backend.GetStringFromPtr(part.ChecksumCRC32)
case types.ChecksumAlgorithmCrc32c:
return backend.GetStringFromPtr(part.ChecksumCRC32C)
case types.ChecksumAlgorithmSha1:
return backend.GetStringFromPtr(part.ChecksumSHA1)
case types.ChecksumAlgorithmSha256:
return backend.GetStringFromPtr(part.ChecksumSHA256)
default:
return ""
}
}
func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
acct, ok := ctx.Value("account").(auth.Account)
if !ok {
@@ -1395,23 +1409,20 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get mp checksums: %w", err)
}
var checksumAlgorithm types.ChecksumAlgorithm
if checksums.Algorithm != "" {
checksumAlgorithm = checksums.Algorithm
}
// if len(checksums.Algorithms) != 0 {
// algorithm := checksums.Algorithms[0]
// ChecksumType should be the same as specified on CreateMultipartUpload
if checksums.Type != input.ChecksumType {
checksumType := checksums.Type
if checksumType == "" {
checksumType = types.ChecksumType("null")
}
// if input.ChecksumCRC32 != nil && algorithm != types.ChecksumAlgorithmCrc32 {
// return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32")
// }
// if input.ChecksumCRC32C != nil && algorithm != types.ChecksumAlgorithmCrc32c {
// return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32c")
// }
// if input.ChecksumSHA1 != nil && algorithm != types.ChecksumAlgorithmSha1 {
// return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-sha1")
// }
// if input.ChecksumSHA256 != nil && algorithm != types.ChecksumAlgorithmSha256 {
// return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-sha256")
// }
// }
return nil, s3err.GetChecksumTypeMismatchOnMpErr(checksumType)
}
// check all parts ok
last := len(parts) - 1
@@ -1457,11 +1468,18 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
}
var hashRdr *utils.HashReader
if len(checksums.Algorithms) != 0 {
hashRdr, err = utils.NewHashReader(nil, "", utils.HashType(strings.ToLower(string(checksums.Algorithms[0]))))
var compositeChecksumRdr *utils.CompositeChecksumReader
switch checksums.Type {
case types.ChecksumTypeFullObject:
hashRdr, err = utils.NewHashReader(nil, "", utils.HashType(strings.ToLower(string(checksumAlgorithm))))
if err != nil {
return nil, fmt.Errorf("initialize hash reader: %w", err)
}
case types.ChecksumTypeComposite:
compositeChecksumRdr, err = utils.NewCompositeChecksumReader(utils.HashType(strings.ToLower(string(checksumAlgorithm))))
if err != nil {
return nil, fmt.Errorf("initialize composite checksum reader: %w", err)
}
}
f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object,
@@ -1487,9 +1505,14 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
}
var rdr io.Reader = pf
if hashRdr != nil {
if checksums.Type == types.ChecksumTypeFullObject {
hashRdr.SetReader(rdr)
rdr = hashRdr
} else if checksums.Type == types.ChecksumTypeComposite {
err := compositeChecksumRdr.Process(getPartChecksum(checksumAlgorithm, part))
if err != nil {
return nil, fmt.Errorf("process %v part checksum: %w", *part.PartNumber, err)
}
}
_, err = io.Copy(f.File(), rdr)
@@ -1594,42 +1617,54 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
var crc32c *string
var sha1 *string
var sha256 *string
var crc64nvme *string
// set checksum
if hashRdr != nil {
algo := checksums.Algorithms[0]
// Calculate, compare with the provided checksum and store them
if checksums.Type != "" {
checksum := s3response.Checksum{
Algorithms: []types.ChecksumAlgorithm{
algo,
},
Algorithm: checksumAlgorithm,
Type: checksums.Type,
}
sum := hashRdr.Sum()
switch hashRdr.Type() {
case utils.HashTypeCRC32:
var sum string
switch checksums.Type {
case types.ChecksumTypeComposite:
sum = compositeChecksumRdr.Sum()
case types.ChecksumTypeFullObject:
sum = hashRdr.Sum()
}
switch checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 != sum {
return nil, s3err.GetChecksumBadDigestErr(algo)
return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm)
}
checksum.CRC32 = &sum
crc32 = &sum
case utils.HashTypeCRC32C:
case types.ChecksumAlgorithmCrc32c:
if input.ChecksumCRC32C != nil && *input.ChecksumCRC32C != sum {
return nil, s3err.GetChecksumBadDigestErr(algo)
return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm)
}
checksum.CRC32C = &sum
crc32c = &sum
case utils.HashTypeSha1:
case types.ChecksumAlgorithmSha1:
if input.ChecksumSHA1 != nil && *input.ChecksumSHA1 != sum {
return nil, s3err.GetChecksumBadDigestErr(algo)
return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm)
}
checksum.SHA1 = &sum
sha1 = &sum
case utils.HashTypeSha256:
case types.ChecksumAlgorithmSha256:
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 != sum {
return nil, s3err.GetChecksumBadDigestErr(algo)
return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm)
}
checksum.SHA256 = &sum
sha256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
if input.ChecksumCRC64NVME != nil && *input.ChecksumCRC64NVME != sum {
return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm)
}
checksum.CRC64NVME = &sum
crc64nvme = &sum
}
err := p.storeChecksums(f.File(), bucket, object, checksum)
if err != nil {
@@ -1669,107 +1704,71 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
os.Remove(filepath.Join(bucket, objdir))
return &s3.CompleteMultipartUploadOutput{
Bucket: &bucket,
ETag: &s3MD5,
Key: &object,
VersionId: &versionID,
ChecksumCRC32: crc32,
ChecksumCRC32C: crc32c,
ChecksumSHA1: sha1,
ChecksumSHA256: sha256,
Bucket: &bucket,
ETag: &s3MD5,
Key: &object,
VersionId: &versionID,
ChecksumCRC32: crc32,
ChecksumCRC32C: crc32c,
ChecksumSHA1: sha1,
ChecksumSHA256: sha256,
ChecksumCRC64NVME: crc64nvme,
ChecksumType: checksums.Type,
}, nil
}
func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart) error {
n := numberOfChecksums(part)
if len(checksum.Algorithms) != 0 {
algo := checksum.Algorithms[0]
if n == 0 {
return s3err.APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("The upload was created using a %v checksum. The complete request must include the checksum for each part. It was missing for part %v in the request.", strings.ToLower(string(algo)), *part.PartNumber),
HTTPStatusCode: http.StatusBadRequest,
}
}
if n > 1 {
return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
}
if part.ChecksumCRC32 != nil {
if ok := utils.IsValidChecksum(*part.ChecksumCRC32, types.ChecksumAlgorithmCrc32); !ok {
return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
}
if *part.ChecksumCRC32 != getString(checksum.CRC32) {
if algo == types.ChecksumAlgorithmCrc32 {
return s3err.GetAPIError(s3err.ErrInvalidPart)
} else {
return s3err.APIError{
Code: "BadDigest",
Description: fmt.Sprintf("The crc32 you specified for part %v did not match what we received.", *part.PartNumber),
HTTPStatusCode: http.StatusBadRequest,
}
}
}
}
if part.ChecksumCRC32C != nil {
if ok := utils.IsValidChecksum(*part.ChecksumCRC32C, types.ChecksumAlgorithmCrc32c); !ok {
return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
}
if *part.ChecksumCRC32C != getString(checksum.CRC32C) {
if algo == types.ChecksumAlgorithmCrc32c {
return s3err.GetAPIError(s3err.ErrInvalidPart)
} else {
return s3err.APIError{
Code: "BadDigest",
Description: fmt.Sprintf("The crc32c you specified for part %v did not match what we received.", *part.PartNumber),
HTTPStatusCode: http.StatusBadRequest,
}
}
}
}
if part.ChecksumSHA1 != nil {
if ok := utils.IsValidChecksum(*part.ChecksumSHA1, types.ChecksumAlgorithmSha1); !ok {
return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
}
if *part.ChecksumSHA1 != getString(checksum.SHA1) {
if algo == types.ChecksumAlgorithmSha1 {
return s3err.GetAPIError(s3err.ErrInvalidPart)
} else {
return s3err.APIError{
Code: "BadDigest",
Description: fmt.Sprintf("The sha1 you specified for part %v did not match what we received.", *part.PartNumber),
HTTPStatusCode: http.StatusBadRequest,
}
}
}
}
if part.ChecksumSHA256 != nil {
if ok := utils.IsValidChecksum(*part.ChecksumSHA256, types.ChecksumAlgorithmSha256); !ok {
return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
}
if *part.ChecksumSHA256 != getString(checksum.SHA256) {
if algo == types.ChecksumAlgorithmSha256 {
return s3err.GetAPIError(s3err.ErrInvalidPart)
} else {
return s3err.APIError{
Code: "BadDigest",
Description: fmt.Sprintf("The sha256 you specified for part %v did not match what we received.", *part.PartNumber),
HTTPStatusCode: http.StatusBadRequest,
}
}
}
if n > 1 {
return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
}
if checksum.Algorithm == "" {
if n != 0 {
return s3err.GetAPIError(s3err.ErrInvalidPart)
}
return nil
}
if n != 0 {
return s3err.GetAPIError(s3err.ErrInvalidPart)
algo := checksum.Algorithm
if n == 0 {
return s3err.APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("The upload was created using a %v checksum. The complete request must include the checksum for each part. It was missing for part %v in the request.", strings.ToLower(string(algo)), *part.PartNumber),
HTTPStatusCode: http.StatusBadRequest,
}
}
for _, cs := range []struct {
checksum *string
expectedChecksum string
algo types.ChecksumAlgorithm
}{
{part.ChecksumCRC32, getString(checksum.CRC32), types.ChecksumAlgorithmCrc32},
{part.ChecksumCRC32C, getString(checksum.CRC32C), types.ChecksumAlgorithmCrc32c},
{part.ChecksumSHA1, getString(checksum.SHA1), types.ChecksumAlgorithmSha1},
{part.ChecksumSHA256, getString(checksum.SHA256), types.ChecksumAlgorithmSha256},
{part.ChecksumCRC64NVME, getString(checksum.CRC64NVME), types.ChecksumAlgorithmCrc64nvme},
} {
if cs.checksum == nil {
continue
}
if !utils.IsValidChecksum(*cs.checksum, cs.algo) {
return s3err.GetAPIError(s3err.ErrInvalidChecksumPart)
}
if *cs.checksum != cs.expectedChecksum {
if algo == cs.algo {
return s3err.GetAPIError(s3err.ErrInvalidPart)
}
return s3err.APIError{
Code: "BadDigest",
Description: fmt.Sprintf("The %v you specified for part %v did not match what we received.", strings.ToLower(string(cs.algo)), *part.PartNumber),
HTTPStatusCode: http.StatusBadRequest,
}
}
}
return nil
@@ -1789,6 +1788,9 @@ func numberOfChecksums(part types.CompletedPart) int {
if getString(part.ChecksumSHA256) != "" {
counter++
}
if getString(part.ChecksumCRC64NVME) != "" {
counter++
}
return counter
}
@@ -1988,17 +1990,13 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
return lmu, fmt.Errorf("get mp checksum: %w", err)
}
var algo types.ChecksumAlgorithm
if len(checksum.Algorithms) != 0 {
algo = checksum.Algorithms[0]
}
uploads = append(uploads, s3response.Upload{
Key: objectName,
UploadID: uploadID,
StorageClass: types.StorageClassStandard,
Initiated: fi.ModTime(),
ChecksumAlgorithm: algo,
ChecksumAlgorithm: checksum.Algorithm,
ChecksumType: checksum.Type,
})
}
}
@@ -2122,11 +2120,6 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
return lpr, fmt.Errorf("get mp checksum: %w", err)
}
var algo types.ChecksumAlgorithm
if len(checksum.Algorithms) != 0 {
algo = checksum.Algorithms[0]
}
var parts []s3response.Part
for _, e := range ents {
pn, err := strconv.Atoi(e.Name())
@@ -2156,14 +2149,15 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
}
parts = append(parts, s3response.Part{
PartNumber: pn,
ETag: etag,
LastModified: fi.ModTime(),
Size: fi.Size(),
ChecksumCRC32: checksum.CRC32,
ChecksumCRC32C: checksum.CRC32C,
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
PartNumber: pn,
ETag: etag,
LastModified: fi.ModTime(),
Size: fi.Size(),
ChecksumCRC32: checksum.CRC32,
ChecksumCRC32C: checksum.CRC32C,
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
ChecksumCRC64NVME: checksum.CRC64NVME,
})
}
@@ -2195,7 +2189,8 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
Parts: parts,
UploadID: uploadID,
StorageClass: types.StorageClassStandard,
ChecksumAlgorithm: algo,
ChecksumAlgorithm: checksum.Algorithm,
ChecksumType: checksum.Type,
}, nil
}
@@ -2290,6 +2285,14 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.
tr = hashRdr
}
if input.ChecksumCRC64NVME != nil {
hashRdr, err = utils.NewHashReader(tr, *input.ChecksumCRC64NVME, utils.HashTypeCRC64NVME)
if err != nil {
return nil, fmt.Errorf("initialize hash reader: %w", err)
}
tr = hashRdr
}
// If only the checksum algorithm is provided register
// a new HashReader to calculate the object checksum
@@ -2309,24 +2312,16 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.
// If checksum isn't provided for the part,
// but it has been provided on mp initalization
if hashRdr == nil && chErr == nil && len(checksums.Algorithms) != 0 {
return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithms[0], "null")
if hashRdr == nil && chErr == nil && checksums.Algorithm != "" {
return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithm, "null")
}
// Check if the provided checksum algorithm match
// the one specified on mp initialization
if hashRdr != nil {
if hashRdr != nil && chErr == nil && checksums.Type != "" {
algo := types.ChecksumAlgorithm(strings.ToUpper(string(hashRdr.Type())))
if chErr != nil {
return nil, s3err.GetChecksumTypeMismatchErr("null", algo)
}
if len(checksums.Algorithms) == 0 {
return nil, s3err.GetChecksumTypeMismatchErr("null", algo)
}
if checksums.Algorithms[0] != algo {
return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithms[0], algo)
if checksums.Algorithm != algo {
return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithm, algo)
}
}
@@ -2349,11 +2344,12 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.
ETag: &etag,
}
// Store the calculated checksum in the object metadata
if hashRdr != nil {
checksum := s3response.Checksum{
Algorithms: []types.ChecksumAlgorithm{input.ChecksumAlgorithm},
Algorithm: input.ChecksumAlgorithm,
}
// Validate the provided checksum
sum := hashRdr.Sum()
switch hashRdr.Type() {
case utils.HashTypeCRC32:
@@ -2368,11 +2364,18 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.
case utils.HashTypeSha256:
checksum.SHA256 = &sum
res.ChecksumSHA256 = &sum
case utils.HashTypeCRC64NVME:
checksum.CRC64NVME = &sum
res.ChecksumCRC64NVME = &sum
}
err := p.storeChecksums(f.File(), bucket, partPath, checksum)
if err != nil {
return nil, fmt.Errorf("store checksum: %w", err)
// Store the checksums if the checksum type has been
// specified on mp initialization
if checksums.Type != "" {
err := p.storeChecksums(f.File(), bucket, partPath, checksum)
if err != nil {
return nil, fmt.Errorf("store checksum: %w", err)
}
}
}
@@ -2519,10 +2522,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyPartResult{}, fmt.Errorf("retreive object part checksums: %w", err)
}
// TODO: Should the checksum be recalculated or just copied ?
var hashRdr *utils.HashReader
if len(mpChecksums.Algorithms) != 0 {
if len(checksums.Algorithms) == 0 || (mpChecksums.Algorithms[0] != checksums.Algorithms[0]) {
hashRdr, err = utils.NewHashReader(tr, "", utils.HashType(strings.ToLower(string(mpChecksums.Algorithms[0]))))
if mpChecksums.Algorithm != "" {
if checksums.Algorithm == "" || mpChecksums.Algorithm != checksums.Algorithm {
hashRdr, err = utils.NewHashReader(tr, "", utils.HashType(strings.ToLower(string(mpChecksums.Algorithm))))
if err != nil {
return s3response.CopyPartResult{}, fmt.Errorf("initialize hash reader: %w", err)
}
@@ -2539,8 +2543,8 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyPartResult{}, fmt.Errorf("copy part data: %w", err)
}
if len(checksums.Algorithms) != 0 {
if len(mpChecksums.Algorithms) == 0 {
if checksums.Algorithm != "" {
if mpChecksums.Algorithm == "" {
checksums = s3response.Checksum{}
} else {
if hashRdr == nil {
@@ -2554,7 +2558,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
if hashRdr != nil {
algo := types.ChecksumAlgorithm(strings.ToUpper(string(hashRdr.Type())))
checksums = s3response.Checksum{
Algorithms: []types.ChecksumAlgorithm{algo},
Algorithm: algo,
}
sum := hashRdr.Sum()
@@ -2567,6 +2571,8 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
checksums.SHA1 = &sum
case types.ChecksumAlgorithmSha256:
checksums.SHA256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
checksums.CRC64NVME = &sum
}
err := p.storeChecksums(f.File(), objPath, "", checksums)
@@ -2600,6 +2606,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
ChecksumCRC32C: checksums.CRC32C,
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
}, nil
}
@@ -2771,6 +2778,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons
rdr = hashRdr
}
if po.ChecksumCRC64NVME != nil {
hashRdr, err = utils.NewHashReader(rdr, *po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
}
rdr = hashRdr
}
// If only the checksum algorithm is provided register
// a new HashReader to calculate the object checksum
@@ -2827,25 +2842,30 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons
}
}
checksum := s3response.Checksum{}
// Store the calculated checksum in the object metadata
if hashRdr != nil {
checksum := s3response.Checksum{
Algorithms: []types.ChecksumAlgorithm{po.ChecksumAlgorithm},
}
// The checksum type is always FULL_OBJECT for PutObject
checksum.Type = types.ChecksumTypeFullObject
sum := hashRdr.Sum()
switch hashRdr.Type() {
case utils.HashTypeCRC32:
checksum.CRC32 = &sum
po.ChecksumCRC32 = &sum
checksum.Algorithm = types.ChecksumAlgorithmCrc32
case utils.HashTypeCRC32C:
checksum.CRC32C = &sum
po.ChecksumCRC32C = &sum
checksum.Algorithm = types.ChecksumAlgorithmCrc32c
case utils.HashTypeSha1:
checksum.SHA1 = &sum
po.ChecksumSHA1 = &sum
checksum.Algorithm = types.ChecksumAlgorithmSha1
case utils.HashTypeSha256:
checksum.SHA256 = &sum
po.ChecksumSHA256 = &sum
checksum.Algorithm = types.ChecksumAlgorithmSha256
case utils.HashTypeCRC64NVME:
checksum.CRC64NVME = &sum
checksum.Algorithm = types.ChecksumAlgorithmCrc64nvme
}
err := p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum)
@@ -2934,12 +2954,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons
}
return s3response.PutObjectOutput{
ETag: etag,
VersionID: versionID,
ChecksumCRC32: po.ChecksumCRC32,
ChecksumCRC32C: po.ChecksumCRC32C,
ChecksumSHA1: po.ChecksumSHA1,
ChecksumSHA256: po.ChecksumSHA256,
ETag: etag,
VersionID: versionID,
ChecksumCRC32: checksum.CRC32,
ChecksumCRC32C: checksum.CRC32C,
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
ChecksumCRC64NVME: checksum.CRC64NVME,
ChecksumType: checksum.Type,
}, nil
}
@@ -3476,11 +3498,16 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO
}
var checksums s3response.Checksum
if input.ChecksumMode == types.ChecksumModeEnabled {
var cType types.ChecksumType
// Skip the checksums retreival if object isn't requested fully
if input.ChecksumMode == types.ChecksumModeEnabled && length-startOffset == objSize {
checksums, err = p.retreiveChecksums(f, bucket, object)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object checksums: %w", err)
}
if checksums.Type != "" {
cType = checksums.Type
}
}
// using an os.File allows zero-copy sendfile via io.Copy(os.File, net.Conn)
@@ -3491,22 +3518,24 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO
}
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: &length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: tagCount,
ContentRange: &contentRange,
StorageClass: types.StorageClassStandard,
VersionId: &versionId,
Body: body,
ChecksumCRC32: checksums.CRC32,
ChecksumCRC32C: checksums.CRC32C,
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
AcceptRanges: &acceptRange,
ContentLength: &length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: tagCount,
ContentRange: &contentRange,
StorageClass: types.StorageClassStandard,
VersionId: &versionId,
Body: body,
ChecksumCRC32: checksums.CRC32,
ChecksumCRC32C: checksums.CRC32C,
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumType: cType,
}, nil
}
@@ -3681,11 +3710,15 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
}
var checksums s3response.Checksum
var cType types.ChecksumType
if input.ChecksumMode == types.ChecksumModeEnabled {
checksums, err = p.retreiveChecksums(nil, bucket, object)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object checksums: %w", err)
}
if checksums.Type != "" {
cType = checksums.Type
}
}
return &s3.HeadObjectOutput{
@@ -3704,6 +3737,8 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
ChecksumCRC32C: checksums.CRC32C,
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumType: cType,
}, nil
}
@@ -3733,10 +3768,12 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr
VersionId: data.VersionId,
DeleteMarker: data.DeleteMarker,
Checksum: &types.Checksum{
ChecksumCRC32: data.ChecksumCRC32,
ChecksumCRC32C: data.ChecksumCRC32C,
ChecksumSHA1: data.ChecksumSHA1,
ChecksumSHA256: data.ChecksumSHA256,
ChecksumCRC32: data.ChecksumCRC32,
ChecksumCRC32C: data.ChecksumCRC32C,
ChecksumSHA1: data.ChecksumSHA1,
ChecksumSHA256: data.ChecksumSHA256,
ChecksumCRC64NVME: data.ChecksumCRC64NVME,
ChecksumType: data.ChecksumType,
},
}, nil
}
@@ -3838,6 +3875,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
var crc32c *string
var sha1 *string
var sha256 *string
var crc64nvme *string
dstObjdPath := filepath.Join(dstBucket, dstObject)
if dstObjdPath == objPath {
@@ -3866,14 +3904,9 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
}
if input.ChecksumAlgorithm != "" {
var algo types.ChecksumAlgorithm
if len(checksums.Algorithms) != 0 {
algo = checksums.Algorithms[0]
}
// If a different checksum algorith is specified
// first caclculate and store the checksum
if algo != input.ChecksumAlgorithm {
if checksums.Algorithm != input.ChecksumAlgorithm {
f, err := os.Open(dstObjdPath)
if err != nil {
return nil, fmt.Errorf("open obj file: %w", err)
@@ -3890,9 +3923,8 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
return nil, fmt.Errorf("read err: %w", err)
}
checksums = s3response.Checksum{
Algorithms: []types.ChecksumAlgorithm{input.ChecksumAlgorithm},
}
checksums = s3response.Checksum{}
sum := hashReader.Sum()
switch hashReader.Type() {
case utils.HashTypeCRC32:
@@ -3907,6 +3939,9 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
case utils.HashTypeSha256:
checksums.SHA256 = &sum
sha256 = &sum
case utils.HashTypeCRC64NVME:
checksums.CRC64NVME = &sum
crc64nvme = &sum
}
err = p.storeChecksums(f, dstBucket, dstObject, checksums)
@@ -3934,11 +3969,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
// If any checksum algorithm is provided, replace, otherwise
// use the existing one
if input.ChecksumAlgorithm != "" {
checksums.Algorithms = []types.ChecksumAlgorithm{input.ChecksumAlgorithm}
}
var chAlgorithm types.ChecksumAlgorithm
if len(checksums.Algorithms) != 0 {
chAlgorithm = checksums.Algorithms[0]
checksums.Algorithm = input.ChecksumAlgorithm
}
res, err := p.PutObject(ctx,
@@ -3948,7 +3979,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
Body: f,
ContentLength: &contentLength,
Metadata: input.Metadata,
ChecksumAlgorithm: chAlgorithm,
ChecksumAlgorithm: checksums.Algorithm,
})
if err != nil {
return nil, err
@@ -3959,6 +3990,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
crc32c = res.ChecksumCRC32C
sha1 = res.ChecksumSHA1
sha256 = res.ChecksumSHA256
crc64nvme = res.ChecksumCRC64NVME
}
fi, err = os.Stat(dstObjdPath)
@@ -3968,12 +4000,13 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
return &s3.CopyObjectOutput{
CopyObjectResult: &types.CopyObjectResult{
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
ChecksumCRC32: crc32,
ChecksumCRC32C: crc32c,
ChecksumSHA1: sha1,
ChecksumSHA256: sha256,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
ChecksumCRC32: crc32,
ChecksumCRC32C: crc32c,
ChecksumSHA1: sha1,
ChecksumSHA256: sha256,
ChecksumCRC64NVME: crc64nvme,
},
VersionId: version,
CopySourceVersionId: &srcVersionId,
@@ -4106,7 +4139,8 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc {
LastModified: &mtime,
Size: &size,
StorageClass: types.ObjectStorageClassStandard,
ChecksumAlgorithm: checksums.Algorithms,
ChecksumAlgorithm: []types.ChecksumAlgorithm{checksums.Algorithm},
ChecksumType: checksums.Type,
}, nil
}
}

56
backend/s3proxy/s3.go
View File

@@ -256,8 +256,10 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
ID: *u.Owner.ID,
DisplayName: *u.Owner.DisplayName,
},
StorageClass: u.StorageClass,
Initiated: *u.Initiated,
StorageClass: u.StorageClass,
Initiated: *u.Initiated,
ChecksumAlgorithm: u.ChecksumAlgorithm,
ChecksumType: u.ChecksumType,
})
}
@@ -293,10 +295,15 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
var parts []s3response.Part
for _, p := range output.Parts {
parts = append(parts, s3response.Part{
PartNumber: int(*p.PartNumber),
LastModified: *p.LastModified,
ETag: *p.ETag,
Size: *p.Size,
PartNumber: int(*p.PartNumber),
LastModified: *p.LastModified,
ETag: *p.ETag,
Size: *p.Size,
ChecksumCRC32: p.ChecksumCRC32,
ChecksumCRC32C: p.ChecksumCRC32C,
ChecksumCRC64NVME: p.ChecksumCRC64NVME,
ChecksumSHA1: p.ChecksumSHA1,
ChecksumSHA256: p.ChecksumSHA256,
})
}
pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -329,6 +336,8 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
MaxParts: int(*output.MaxParts),
IsTruncated: *output.IsTruncated,
Parts: parts,
ChecksumAlgorithm: output.ChecksumAlgorithm,
ChecksumType: output.ChecksumType,
}, nil
}
@@ -348,8 +357,13 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
}
return s3response.CopyPartResult{
LastModified: *output.CopyPartResult.LastModified,
ETag: output.CopyPartResult.ETag,
LastModified: *output.CopyPartResult.LastModified,
ETag: output.CopyPartResult.ETag,
ChecksumCRC32: output.CopyPartResult.ChecksumCRC32,
ChecksumCRC32C: output.CopyPartResult.ChecksumCRC32C,
ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME,
ChecksumSHA1: output.CopyPartResult.ChecksumSHA1,
ChecksumSHA256: output.CopyPartResult.ChecksumSHA256,
}, nil
}
@@ -369,8 +383,13 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3re
}
return s3response.PutObjectOutput{
ETag: *output.ETag,
VersionID: versionID,
ETag: *output.ETag,
VersionID: versionID,
ChecksumCRC32: output.ChecksumCRC32,
ChecksumCRC32C: output.ChecksumCRC32C,
ChecksumCRC64NVME: output.ChecksumCRC64NVME,
ChecksumSHA1: output.ChecksumSHA1,
ChecksumSHA256: output.ChecksumSHA256,
}, nil
}
@@ -421,6 +440,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
ObjectSize: out.ObjectSize,
StorageClass: out.StorageClass,
ObjectParts: &parts,
Checksum: out.Checksum,
}, handleError(err)
}
@@ -833,13 +853,15 @@ func convertObjects(objs []types.Object) []s3response.Object {
for _, obj := range objs {
result = append(result, s3response.Object{
ETag: obj.ETag,
Key: obj.Key,
LastModified: obj.LastModified,
Owner: obj.Owner,
Size: obj.Size,
RestoreStatus: obj.RestoreStatus,
StorageClass: obj.StorageClass,
ETag: obj.ETag,
Key: obj.Key,
LastModified: obj.LastModified,
Owner: obj.Owner,
Size: obj.Size,
RestoreStatus: obj.RestoreStatus,
StorageClass: obj.StorageClass,
ChecksumAlgorithm: obj.ChecksumAlgorithm,
ChecksumType: obj.ChecksumType,
})
}

75
s3api/controllers/base.go
View File

@@ -607,6 +607,18 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumCRC64NVME != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
}
if res.ChecksumType != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}
// Set x-amz-meta-... headers
utils.SetMetaHeaders(ctx, res.Metadata)
@@ -2079,6 +2091,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
})
if err == nil {
headers := []utils.CustomHeader{}
@@ -2112,6 +2125,12 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumCRC64NVME != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
}
utils.SetResponseHeaders(ctx, headers)
}
@@ -2511,6 +2530,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
})
if err != nil {
return SendResponse(ctx, err,
@@ -2562,6 +2582,18 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Value: getstring(res.ChecksumSHA256),
})
}
if getstring(res.ChecksumCRC64NVME) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: getstring(res.ChecksumCRC64NVME),
})
}
if res.ChecksumType != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}
utils.SetResponseHeaders(ctx, hdrs)
@@ -3206,6 +3238,18 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumCRC64NVME != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
}
if res.ChecksumType != "" {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}
contentType := getstring(res.ContentType)
if contentType == "" {
@@ -3430,6 +3474,21 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
})
}
checksumType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
err = utils.IsChecksumTypeValid(checksumType)
if err != nil {
if c.debug {
log.Printf("invalid checksum type: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCompleteMultipartUpload,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.CompleteMultipartUpload(ctx.Context(),
&s3.CompleteMultipartUploadInput{
Bucket: &bucket,
@@ -3438,10 +3497,12 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
MultipartUpload: &types.CompletedMultipartUpload{
Parts: data.Parts,
},
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumType: checksumType,
})
if err == nil {
if getstring(res.VersionId) != "" {
@@ -3507,11 +3568,10 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
metadata := utils.GetUserMetaData(&ctx.Request().Header)
checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
err = utils.IsChecksumAlgorithmValid(checksumAlgorithm)
checksumAlgorithm, checksumType, err := utils.ParseCreateMpChecksumHeaders(ctx)
if err != nil {
if c.debug {
log.Printf("invalid checksum algorithm: %v", checksumAlgorithm)
log.Printf("err parsing checksum headers: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
@@ -3534,6 +3594,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus,
Metadata: metadata,
ChecksumAlgorithm: checksumAlgorithm,
ChecksumType: checksumType,
})
if err == nil {
if checksumAlgorithm != "" {

70
s3api/utils/csum-reader.go
View File

@@ -21,9 +21,12 @@ import (
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"math/bits"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
@@ -45,6 +48,8 @@ const (
HashTypeCRC32 HashType = "crc32"
// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
HashTypeCRC32C HashType = "crc32c"
// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
HashTypeCRC64NVME HashType = "crc64nvme"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone HashType = "none"
)
@@ -83,6 +88,8 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
hash = crc32.NewIEEE()
case HashTypeCRC32C:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeCRC64NVME:
hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case HashTypeNone:
hash = noop{}
default:
@@ -136,6 +143,11 @@ func (hr *HashReader) Read(p []byte) (int, error) {
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
}
case HashTypeCRC64NVME:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
}
default:
return n, errInvalidHashType
}
@@ -162,6 +174,8 @@ func (hr *HashReader) Sum() string {
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC64NVME:
return Base64SumString(hr.hash.Sum(nil))
default:
return ""
}
@@ -183,3 +197,59 @@ func (n noop) Sum(b []byte) []byte { return []byte{} }
func (n noop) Reset() {}
func (n noop) Size() int { return 0 }
func (n noop) BlockSize() int { return 1 }
// NewCompositeChecksumReader initializes a composite checksum
// processor, which decodes and validates the provided
// checksums and returns the final checksum based on
// the previous processings.
//
// The supported checksum types are:
// - CRC32
// - CRC32C
// - SHA1
// - SHA256
func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
var hasher hash.Hash
switch ht {
case HashTypeSha256:
hasher = sha256.New()
case HashTypeSha1:
hasher = sha1.New()
case HashTypeCRC32:
hasher = crc32.NewIEEE()
case HashTypeCRC32C:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeNone:
hasher = noop{}
default:
return nil, errInvalidHashType
}
return &CompositeChecksumReader{
hasher: hasher,
}, nil
}
type CompositeChecksumReader struct {
hasher hash.Hash
}
// Decodes and writes the checksum in the hasher
func (ccr *CompositeChecksumReader) Process(checksum string) error {
data, err := base64.StdEncoding.DecodeString(checksum)
if err != nil {
return fmt.Errorf("base64 decode: %w", err)
}
_, err = ccr.hasher.Write(data)
if err != nil {
return fmt.Errorf("hash write: %w", err)
}
return nil
}
// Returns the base64 encoded composite checksum
func (ccr *CompositeChecksumReader) Sum() string {
return Base64SumString(ccr.hasher.Sum(nil))
}

116
s3api/utils/utils.go
View File

@@ -461,7 +461,7 @@ func shouldEscape(c byte) bool {
}
func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) {
sdkAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-sdk-checksum-algorithm"))
sdkAlgorithm := types.ChecksumAlgorithm(ctx.Get("X-Amz-Sdk-Checksum-Algorithm"))
err := IsChecksumAlgorithmValid(sdkAlgorithm)
if err != nil {
@@ -469,10 +469,11 @@ func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.Ch
}
checksums := map[types.ChecksumAlgorithm]string{
types.ChecksumAlgorithmCrc32: ctx.Get("x-amz-checksum-crc32"),
types.ChecksumAlgorithmCrc32c: ctx.Get("x-amz-checksum-crc32c"),
types.ChecksumAlgorithmSha1: ctx.Get("x-amz-checksum-sha1"),
types.ChecksumAlgorithmSha256: ctx.Get("x-amz-checksum-sha256"),
types.ChecksumAlgorithmCrc32: ctx.Get("X-Amz-Checksum-Crc32"),
types.ChecksumAlgorithmCrc32c: ctx.Get("X-Amz-Checksum-Crc32c"),
types.ChecksumAlgorithmSha1: ctx.Get("X-Amz-Checksum-Sha1"),
types.ChecksumAlgorithmSha256: ctx.Get("X-Amz-Checksum-Sha256"),
types.ChecksumAlgorithmCrc64nvme: ctx.Get("X-Amz-Checksum-Crc64nvme"),
}
headerCtr := 0
@@ -487,6 +488,7 @@ func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.Ch
return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
}
if val != "" {
sdkAlgorithm = al
headerCtr++
}
@@ -499,10 +501,11 @@ func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.Ch
}
var checksumLengths = map[types.ChecksumAlgorithm]int{
types.ChecksumAlgorithmCrc32: 4,
types.ChecksumAlgorithmCrc32c: 4,
types.ChecksumAlgorithmSha1: 20,
types.ChecksumAlgorithmSha256: 32,
types.ChecksumAlgorithmCrc32: 4,
types.ChecksumAlgorithmCrc32c: 4,
types.ChecksumAlgorithmCrc64nvme: 8,
types.ChecksumAlgorithmSha1: 20,
types.ChecksumAlgorithmSha256: 32,
}
func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
@@ -524,9 +527,102 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
alg != types.ChecksumAlgorithmCrc32 &&
alg != types.ChecksumAlgorithmCrc32c &&
alg != types.ChecksumAlgorithmSha1 &&
alg != types.ChecksumAlgorithmSha256 {
alg != types.ChecksumAlgorithmSha256 &&
alg != types.ChecksumAlgorithmCrc64nvme {
return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
}
return nil
}
// Validates the provided checksum type
func IsChecksumTypeValid(t types.ChecksumType) error {
if t != "" &&
t != types.ChecksumTypeComposite &&
t != types.ChecksumTypeFullObject {
return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")
}
return nil
}
type checksumTypeSchema map[types.ChecksumType]struct{}
type checksumSchema map[types.ChecksumAlgorithm]checksumTypeSchema
// A table defining the checksum algorithm/type support
var checksumMap checksumSchema = checksumSchema{
types.ChecksumAlgorithmCrc32: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmCrc32c: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmSha1: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmSha256: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmCrc64nvme: checksumTypeSchema{
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
// Both could be empty
"": checksumTypeSchema{
"": struct{}{},
},
}
// Checks if checksum type and algorithm are supported together
func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType) error {
typeSchema := checksumMap[algo]
_, ok := typeSchema[t]
if !ok {
return s3err.GetChecksumSchemaMismatchErr(algo, t)
}
return nil
}
// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers
func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) {
algo := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
if err := IsChecksumAlgorithmValid(algo); err != nil {
return "", "", err
}
chType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
if err := IsChecksumTypeValid(chType); err != nil {
return "", "", err
}
// Verify if checksum algorithm is provided, if
// checksum type is specified
if chType != "" && algo == "" {
return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)
}
// Verify if the checksum type is supported for
// the provided checksum algorithm
if err := checkChecksumTypeAndAlgo(algo, chType); err != nil {
return algo, chType, err
}
// x-amz-checksum-type defaults to COMPOSITE
// if x-amz-checksum-algorithm is set except
// for the CRC64NVME algorithm: it defaults to FULL_OBJECT
if algo != "" && chType == "" {
if algo == types.ChecksumAlgorithmCrc64nvme {
chType = types.ChecksumTypeFullObject
} else {
chType = types.ChecksumTypeComposite
}
}
return algo, chType, nil
}

169
s3api/utils/utils_test.go
View File

@@ -572,6 +572,13 @@ func TestIsChecksumAlgorithmValid(t *testing.T) {
},
wantErr: false,
},
{
name: "crc64nvme",
args: args{
alg: types.ChecksumAlgorithmCrc64nvme,
},
wantErr: false,
},
{
name: "invalid",
args: args{
@@ -680,3 +687,165 @@ func TestIsValidChecksum(t *testing.T) {
})
}
}
func TestIsChecksumTypeValid(t *testing.T) {
type args struct {
t types.ChecksumType
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "valid_FULL_OBJECT",
args: args{
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "valid_COMPOSITE",
args: args{
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "invalid",
args: args{
t: types.ChecksumType("invalid_checksum_type"),
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := IsChecksumTypeValid(tt.args.t); (err != nil) != tt.wantErr {
t.Errorf("IsChecksumTypeValid() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func Test_checkChecksumTypeAndAlgo(t *testing.T) {
type args struct {
algo types.ChecksumAlgorithm
t types.ChecksumType
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "full_object-crc32",
args: args{
algo: types.ChecksumAlgorithmCrc32,
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "full_object-crc32c",
args: args{
algo: types.ChecksumAlgorithmCrc32c,
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "full_object-sha1",
args: args{
algo: types.ChecksumAlgorithmSha1,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-sha256",
args: args{
algo: types.ChecksumAlgorithmSha1,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-crc64nvme",
args: args{
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "full_object-crc32",
args: args{
algo: types.ChecksumAlgorithmCrc32,
t: types.ChecksumTypeFullObject,
},
wantErr: false,
},
{
name: "composite-crc32",
args: args{
algo: types.ChecksumAlgorithmCrc32,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-crc32c",
args: args{
algo: types.ChecksumAlgorithmCrc32c,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-sha1",
args: args{
algo: types.ChecksumAlgorithmSha1,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-sha256",
args: args{
algo: types.ChecksumAlgorithmSha256,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-crc64nvme",
args: args{
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeComposite,
},
wantErr: true,
},
{
name: "composite-empty",
args: args{
t: types.ChecksumTypeComposite,
},
wantErr: true,
},
{
name: "full_object-empty",
args: args{
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := checkChecksumTypeAndAlgo(tt.args.algo, tt.args.t); (err != nil) != tt.wantErr {
t.Errorf("checkChecksumTypeAndAlgo() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

26
s3err/s3err.go
View File

@@ -147,6 +147,7 @@ const (
ErrMultipleChecksumHeaders
ErrInvalidChecksumAlgorithm
ErrInvalidChecksumPart
ErrChecksumTypeWithAlgo
// Non-AWS errors
ErrExistingObjectIsDirectory
@@ -606,6 +607,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Invalid Base64 or multiple checksums present in request",
HTTPStatusCode: http.StatusBadRequest,
},
ErrChecksumTypeWithAlgo: {
Code: "InvalidRequest",
Description: "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.",
HTTPStatusCode: http.StatusBadRequest,
},
// non aws errors
ErrExistingObjectIsDirectory: {
@@ -719,7 +725,7 @@ func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIErr
}
}
// Return incorrect checksum APIError
// Returns incorrect checksum APIError
func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError {
return APIError{
Code: "BadDigest",
@@ -727,3 +733,21 @@ func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError {
HTTPStatusCode: http.StatusBadRequest,
}
}
// Returns checksum type mismatch error with checksum algorithm
func GetChecksumSchemaMismatchErr(algo types.ChecksumAlgorithm, t types.ChecksumType) APIError {
return APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("The %v checksum type cannot be used with the %v checksum algorithm.", algo, strings.ToLower(string(t))),
HTTPStatusCode: http.StatusBadRequest,
}
}
// Returns checksum type mismatch error for multipart uploads
func GetChecksumTypeMismatchOnMpErr(t types.ChecksumType) APIError {
return APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("The upload was created using the %v checksum mode. The complete request must use the same checksum mode.", t),
HTTPStatusCode: http.StatusBadRequest,
}
}

61
s3response/s3response.go
View File

@@ -29,24 +29,27 @@ const (
)
type PutObjectOutput struct {
ETag string
VersionID string
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ETag string
VersionID string
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumType types.ChecksumType
}
// Part describes part metadata.
type Part struct {
PartNumber int
LastModified time.Time
ETag string
Size int64
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
PartNumber int
LastModified time.Time
ETag string
Size int64
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
}
func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -71,6 +74,7 @@ type ListPartsResult struct {
Key string
UploadID string `xml:"UploadId"`
ChecksumAlgorithm types.ChecksumAlgorithm
ChecksumType types.ChecksumType
Initiator Initiator
Owner Owner
@@ -180,6 +184,7 @@ type ListObjectsV2Result struct {
type Object struct {
ChecksumAlgorithm []types.ChecksumAlgorithm
ChecksumType types.ChecksumType
ETag *string
Key *string
LastModified *time.Time
@@ -215,6 +220,7 @@ type Upload struct {
StorageClass types.StorageClass
Initiated time.Time
ChecksumAlgorithm types.ChecksumAlgorithm
ChecksumType types.ChecksumType
}
func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -345,13 +351,14 @@ type CopyObjectResult struct {
}
type CopyPartResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
LastModified time.Time
ETag *string
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
LastModified time.Time
ETag *string
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
// not included in the body
CopySourceVersionId string `xml:"-"`
@@ -494,10 +501,12 @@ type ListBucketsResult struct {
}
type Checksum struct {
Algorithms []types.ChecksumAlgorithm
Algorithm types.ChecksumAlgorithm
Type types.ChecksumType
CRC32 *string
CRC32C *string
SHA1 *string
SHA256 *string
CRC32 *string
CRC32C *string
SHA1 *string
SHA256 *string
CRC64NVME *string
}

945
tests/integration/group-tests.go
View File

File diff suppressed because it is too large Load Diff

973
tests/integration/tests.go
View File

File diff suppressed because it is too large Load Diff

82
tests/integration/utils.go
View File

@@ -481,6 +481,7 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client)
type mpCfg struct {
checksumAlgorithm types.ChecksumAlgorithm
checksumType types.ChecksumType
}
type mpOpt func(*mpCfg)
@@ -488,6 +489,9 @@ type mpOpt func(*mpCfg)
func withChecksum(algo types.ChecksumAlgorithm) mpOpt {
return func(mc *mpCfg) { mc.checksumAlgorithm = algo }
}
func withChecksumType(t types.ChecksumType) mpOpt {
return func(mc *mpCfg) { mc.checksumType = t }
}
func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) {
cfg := new(mpCfg)
@@ -499,6 +503,7 @@ func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.Creat
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: cfg.checksumAlgorithm,
ChecksumType: cfg.checksumType,
})
cancel()
return out, err
@@ -535,6 +540,9 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
if item.ChecksumAlgorithm != list2[i].ChecksumAlgorithm {
return false
}
if item.ChecksumType != list2[i].ChecksumType {
return false
}
}
return true
@@ -542,30 +550,52 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
func compareParts(parts1, parts2 []types.Part) bool {
if len(parts1) != len(parts2) {
fmt.Printf("list length are not equal: %v != %v\n", len(parts1), len(parts2))
return false
}
for i, prt := range parts1 {
if *prt.PartNumber != *parts2[i].PartNumber {
fmt.Printf("partNumbers are not equal, %v != %v\n", *prt.PartNumber, *parts2[i].PartNumber)
return false
}
if *prt.ETag != *parts2[i].ETag {
fmt.Printf("etags are not equal, %v != %v\n", *prt.ETag, *parts2[i].ETag)
return false
}
if *prt.Size != *parts2[i].Size {
fmt.Printf("sizes are not equal, %v != %v\n", *prt.Size, *parts2[i].Size)
return false
}
if getString(prt.ChecksumCRC32) != getString(parts2[i].ChecksumCRC32) {
return false
if prt.ChecksumCRC32 != nil {
if *prt.ChecksumCRC32 != getString(parts2[i].ChecksumCRC32) {
fmt.Printf("crc32 checksums are not equal, %v != %v\n", *prt.ChecksumCRC32, getString(parts2[i].ChecksumCRC32))
return false
}
}
if getString(prt.ChecksumCRC32C) != getString(parts2[i].ChecksumCRC32C) {
return false
if prt.ChecksumCRC32C != nil {
if *prt.ChecksumCRC32C != getString(parts2[i].ChecksumCRC32C) {
fmt.Printf("crc32c checksums are not equal, %v != %v\n", *prt.ChecksumCRC32C, getString(parts2[i].ChecksumCRC32C))
return false
}
}
if getString(prt.ChecksumSHA1) != getString(parts2[i].ChecksumSHA1) {
return false
if prt.ChecksumSHA1 != nil {
if *prt.ChecksumSHA1 != getString(parts2[i].ChecksumSHA1) {
fmt.Printf("sha1 checksums are not equal, %v != %v\n", *prt.ChecksumSHA1, getString(parts2[i].ChecksumSHA1))
return false
}
}
if getString(prt.ChecksumSHA256) != getString(parts2[i].ChecksumSHA256) {
return false
if prt.ChecksumSHA256 != nil {
if *prt.ChecksumSHA256 != getString(parts2[i].ChecksumSHA256) {
fmt.Printf("sha256 checksums are not equal, %v != %v\n", *prt.ChecksumSHA256, getString(parts2[i].ChecksumSHA256))
return false
}
}
if prt.ChecksumCRC64NVME != nil {
if *prt.ChecksumCRC64NVME != getString(parts2[i].ChecksumCRC64NVME) {
fmt.Printf("crc64nvme checksums are not equal, %v != %v\n", *prt.ChecksumCRC64NVME, getString(parts2[i].ChecksumCRC64NVME))
return false
}
}
}
return true
@@ -691,6 +721,13 @@ func compareObjects(list1, list2 []types.Object) bool {
return false
}
}
if obj.ChecksumType != "" {
if obj.ChecksumType[0] != list2[i].ChecksumType[0] {
fmt.Printf("checksum types are not equal: (%q %q) %v != %v\n",
*obj.Key, *list2[i].Key, obj.ChecksumType[0], list2[i].ChecksumType[0])
return false
}
}
}
return true
@@ -804,15 +841,26 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
return parts, "", err
}
parts = append(parts, types.Part{
ETag: out.ETag,
PartNumber: &pn,
Size: &partSize,
ChecksumCRC32: out.ChecksumCRC32,
ChecksumCRC32C: out.ChecksumCRC32C,
ChecksumSHA1: out.ChecksumSHA1,
ChecksumSHA256: out.ChecksumSHA256,
})
part := types.Part{
ETag: out.ETag,
PartNumber: &pn,
Size: &partSize,
}
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
part.ChecksumCRC32 = out.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
part.ChecksumCRC32C = out.ChecksumCRC32C
case types.ChecksumAlgorithmSha1:
part.ChecksumSHA1 = out.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
part.ChecksumSHA256 = out.ChecksumSHA256
case types.ChecksumAlgorithmCrc64nvme:
part.ChecksumCRC64NVME = out.ChecksumCRC64NVME
}
parts = append(parts, part)
}
sum := hash.Sum(nil)