mirror of
https://github.com/versity/versitygw.git
synced 2026-08-20 06:06:19 +00:00
Merge pull request #1894 from versity/sis/getobject-directory-object-checksum
feat: adds checksums for directory objects in posix
This commit is contained in:
+129
-84
@@ -3052,6 +3052,24 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getEmptyChecksumValue returns the base64-encoded checksum
|
||||
// for an empty payload for the given algorithm defaulting to crc64nvme
|
||||
func getEmptyChecksumValue(algo types.ChecksumAlgorithm) string {
|
||||
switch algo {
|
||||
case types.ChecksumAlgorithmCrc32:
|
||||
return "AAAAAA=="
|
||||
case types.ChecksumAlgorithmCrc32c:
|
||||
return "AAAAAA=="
|
||||
case types.ChecksumAlgorithmSha1:
|
||||
return "2jmj7l5rSw0yVb/vlWAYkK/YBwk="
|
||||
case types.ChecksumAlgorithmSha256:
|
||||
return "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
|
||||
default:
|
||||
// default to crc64nvme
|
||||
return "AAAAAAAAAAA="
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
|
||||
release, err := p.acquireActionSlot(ctx)
|
||||
if err != nil {
|
||||
@@ -3104,6 +3122,39 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
if po.ContentLength != nil {
|
||||
contentLength = *po.ContentLength
|
||||
}
|
||||
|
||||
chRdr, chunkUpload := po.Body.(middlewares.ChecksumReader)
|
||||
isTrailingChecksum := chunkUpload && chRdr.Algorithm() != ""
|
||||
|
||||
checksumAlgorithm := po.ChecksumAlgorithm
|
||||
checksumValue := ""
|
||||
|
||||
if isTrailingChecksum {
|
||||
checksumAlgorithm = types.ChecksumAlgorithm(strings.ToUpper(chRdr.Algorithm()))
|
||||
checksumValue = chRdr.Checksum()
|
||||
} else {
|
||||
hashConfigs := []hashConfig{
|
||||
{po.ChecksumCRC32, utils.HashTypeCRC32},
|
||||
{po.ChecksumCRC32C, utils.HashTypeCRC32C},
|
||||
{po.ChecksumSHA1, utils.HashTypeSha1},
|
||||
{po.ChecksumSHA256, utils.HashTypeSha256},
|
||||
{po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME},
|
||||
}
|
||||
|
||||
for _, config := range hashConfigs {
|
||||
if config.value != nil {
|
||||
checksumAlgorithm = types.ChecksumAlgorithm(strings.ToUpper(string(config.hashType)))
|
||||
checksumValue = *config.value
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if checksumAlgorithm == "" {
|
||||
// default to crc64nvme
|
||||
checksumAlgorithm = types.ChecksumAlgorithmCrc64nvme
|
||||
}
|
||||
}
|
||||
|
||||
if strings.HasSuffix(*po.Key, "/") {
|
||||
// object is directory
|
||||
if contentLength != 0 {
|
||||
@@ -3143,10 +3194,46 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err)
|
||||
}
|
||||
|
||||
expectedSum := getEmptyChecksumValue(checksumAlgorithm)
|
||||
if checksumValue != "" && expectedSum != checksumValue {
|
||||
return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm)
|
||||
}
|
||||
|
||||
// set empty checksum
|
||||
checksum := s3response.Checksum{
|
||||
Type: types.ChecksumTypeFullObject,
|
||||
Algorithm: checksumAlgorithm,
|
||||
}
|
||||
|
||||
// Store the calculated checksum in the object metadata
|
||||
switch checksumAlgorithm {
|
||||
case types.ChecksumAlgorithmCrc32:
|
||||
checksum.CRC32 = &expectedSum
|
||||
case types.ChecksumAlgorithmCrc32c:
|
||||
checksum.CRC32C = &expectedSum
|
||||
case types.ChecksumAlgorithmSha1:
|
||||
checksum.SHA1 = &expectedSum
|
||||
case types.ChecksumAlgorithmSha256:
|
||||
checksum.SHA256 = &expectedSum
|
||||
case types.ChecksumAlgorithmCrc64nvme:
|
||||
checksum.CRC64NVME = &expectedSum
|
||||
}
|
||||
|
||||
err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
|
||||
}
|
||||
|
||||
// for directory object no version is created
|
||||
return s3response.PutObjectOutput{
|
||||
ETag: emptyMD5,
|
||||
Size: &contentLength,
|
||||
ETag: emptyMD5,
|
||||
Size: &contentLength,
|
||||
ChecksumType: checksum.Type,
|
||||
ChecksumCRC32: checksum.CRC32,
|
||||
ChecksumCRC32C: checksum.CRC32C,
|
||||
ChecksumCRC64NVME: checksum.CRC64NVME,
|
||||
ChecksumSHA1: checksum.SHA1,
|
||||
ChecksumSHA256: checksum.SHA256,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -3208,53 +3295,13 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
rdr := io.TeeReader(po.Body, hash)
|
||||
|
||||
var hashRdr *utils.HashReader
|
||||
|
||||
chRdr, chunkUpload := po.Body.(middlewares.ChecksumReader)
|
||||
isTrailingChecksum := chunkUpload && chRdr.Algorithm() != ""
|
||||
|
||||
if !isTrailingChecksum {
|
||||
hashConfigs := []hashConfig{
|
||||
{po.ChecksumCRC32, utils.HashTypeCRC32},
|
||||
{po.ChecksumCRC32C, utils.HashTypeCRC32C},
|
||||
{po.ChecksumSHA1, utils.HashTypeSha1},
|
||||
{po.ChecksumSHA256, utils.HashTypeSha256},
|
||||
{po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME},
|
||||
hashRdr, err = utils.NewHashReader(rdr, checksumValue, utils.HashType(strings.ToLower(string(checksumAlgorithm))))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
|
||||
}
|
||||
|
||||
for _, config := range hashConfigs {
|
||||
if config.value != nil {
|
||||
hashRdr, err = utils.NewHashReader(rdr, *config.value, config.hashType)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
|
||||
}
|
||||
|
||||
rdr = hashRdr
|
||||
}
|
||||
}
|
||||
|
||||
// If only the checksum algorithm is provided register
|
||||
// a new HashReader to calculate the object checksum
|
||||
// This can never happen with PutObject direct call
|
||||
// it's there for CopyObject to add a new checksum
|
||||
if hashRdr == nil && po.ChecksumAlgorithm != "" {
|
||||
hashRdr, err = utils.NewHashReader(rdr, "", utils.HashType(strings.ToLower(string(po.ChecksumAlgorithm))))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
|
||||
}
|
||||
|
||||
rdr = hashRdr
|
||||
}
|
||||
|
||||
if hashRdr == nil {
|
||||
// if no precalculated checksum or sdk checksum algorithm is provided
|
||||
// and no streaming upload has checksum, default to crc64nvme
|
||||
hashRdr, err = utils.NewHashReader(rdr, "", utils.HashTypeCRC64NVME)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
|
||||
}
|
||||
|
||||
rdr = hashRdr
|
||||
}
|
||||
rdr = hashRdr
|
||||
}
|
||||
|
||||
_, err = io.Copy(f, rdr)
|
||||
@@ -3304,43 +3351,35 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
}
|
||||
}
|
||||
|
||||
var chAlgo utils.HashType
|
||||
var sum string
|
||||
|
||||
if isTrailingChecksum {
|
||||
chAlgo = utils.HashType(chRdr.Algorithm())
|
||||
sum = chRdr.Checksum()
|
||||
} else if hashRdr != nil {
|
||||
chAlgo = hashRdr.Type()
|
||||
} else {
|
||||
sum = hashRdr.Sum()
|
||||
}
|
||||
|
||||
checksum := s3response.Checksum{}
|
||||
checksum := s3response.Checksum{
|
||||
Type: types.ChecksumTypeFullObject,
|
||||
Algorithm: checksumAlgorithm,
|
||||
}
|
||||
|
||||
// Store the calculated checksum in the object metadata
|
||||
if sum != "" {
|
||||
checksum.Type = types.ChecksumTypeFullObject
|
||||
switch chAlgo {
|
||||
case utils.HashTypeCRC32:
|
||||
checksum.CRC32 = &sum
|
||||
checksum.Algorithm = types.ChecksumAlgorithmCrc32
|
||||
case utils.HashTypeCRC32C:
|
||||
checksum.CRC32C = &sum
|
||||
checksum.Algorithm = types.ChecksumAlgorithmCrc32c
|
||||
case utils.HashTypeSha1:
|
||||
checksum.SHA1 = &sum
|
||||
checksum.Algorithm = types.ChecksumAlgorithmSha1
|
||||
case utils.HashTypeSha256:
|
||||
checksum.SHA256 = &sum
|
||||
checksum.Algorithm = types.ChecksumAlgorithmSha256
|
||||
case utils.HashTypeCRC64NVME:
|
||||
checksum.CRC64NVME = &sum
|
||||
checksum.Algorithm = types.ChecksumAlgorithmCrc64nvme
|
||||
}
|
||||
err := p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
|
||||
}
|
||||
switch checksumAlgorithm {
|
||||
case types.ChecksumAlgorithmCrc32:
|
||||
checksum.CRC32 = &sum
|
||||
case types.ChecksumAlgorithmCrc32c:
|
||||
checksum.CRC32C = &sum
|
||||
case types.ChecksumAlgorithmSha1:
|
||||
checksum.SHA1 = &sum
|
||||
case types.ChecksumAlgorithmSha256:
|
||||
checksum.SHA256 = &sum
|
||||
case types.ChecksumAlgorithmCrc64nvme:
|
||||
checksum.CRC64NVME = &sum
|
||||
}
|
||||
err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
|
||||
}
|
||||
|
||||
err = p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, etagkey, []byte(etag))
|
||||
@@ -4000,8 +4039,22 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
tagCount = &tgCount
|
||||
}
|
||||
|
||||
var checksums s3response.Checksum
|
||||
if input.ChecksumMode == types.ChecksumModeEnabled {
|
||||
checksums, err = p.retrieveChecksums(nil, bucket, object)
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
return nil, fmt.Errorf("get object checksums: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var length int64 = 0
|
||||
return &s3.GetObjectOutput{
|
||||
ChecksumCRC32: checksums.CRC32,
|
||||
ChecksumCRC32C: checksums.CRC32C,
|
||||
ChecksumSHA1: checksums.SHA1,
|
||||
ChecksumSHA256: checksums.SHA256,
|
||||
ChecksumCRC64NVME: checksums.CRC64NVME,
|
||||
ChecksumType: checksums.Type,
|
||||
AcceptRanges: backend.GetPtrFromString("bytes"),
|
||||
ContentLength: &length,
|
||||
ContentEncoding: objMeta.ContentEncoding,
|
||||
@@ -4081,16 +4134,12 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
}
|
||||
|
||||
var checksums s3response.Checksum
|
||||
var cType types.ChecksumType
|
||||
// Skip the checksums retreival if object isn't requested fully
|
||||
if input.ChecksumMode == types.ChecksumModeEnabled && length-startOffset == objSize {
|
||||
checksums, err = p.retrieveChecksums(f, bucket, object)
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
return nil, fmt.Errorf("get object checksums: %w", err)
|
||||
}
|
||||
if checksums.Type != "" {
|
||||
cType = checksums.Type
|
||||
}
|
||||
}
|
||||
|
||||
// using an os.File allows zero-copy sendfile via io.Copy(os.File, net.Conn)
|
||||
@@ -4122,7 +4171,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
ChecksumSHA1: checksums.SHA1,
|
||||
ChecksumSHA256: checksums.SHA256,
|
||||
ChecksumCRC64NVME: checksums.CRC64NVME,
|
||||
ChecksumType: cType,
|
||||
ChecksumType: checksums.Type,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -4290,15 +4339,11 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
|
||||
}
|
||||
|
||||
var checksums s3response.Checksum
|
||||
var cType types.ChecksumType
|
||||
if input.ChecksumMode == types.ChecksumModeEnabled {
|
||||
checksums, err = p.retrieveChecksums(nil, bucket, object)
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
return nil, fmt.Errorf("get object checksums: %w", err)
|
||||
}
|
||||
if checksums.Type != "" {
|
||||
cType = checksums.Type
|
||||
}
|
||||
}
|
||||
|
||||
var tagCount *int32
|
||||
@@ -4334,7 +4379,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
|
||||
ChecksumSHA1: checksums.SHA1,
|
||||
ChecksumSHA256: checksums.SHA256,
|
||||
ChecksumCRC64NVME: checksums.CRC64NVME,
|
||||
ChecksumType: cType,
|
||||
ChecksumType: checksums.Type,
|
||||
TagCount: tagCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -308,6 +308,90 @@ func GetObject_checksums(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func GetObject_dir_object_checksum(s *S3Conf) error {
|
||||
testName := "GetObject_dir_object_checksum"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
for i, obj := range []struct {
|
||||
key string
|
||||
expectedSum string
|
||||
checksumAlgo types.ChecksumAlgorithm
|
||||
}{
|
||||
{
|
||||
key: "obj-1/",
|
||||
expectedSum: "AAAAAA==",
|
||||
checksumAlgo: types.ChecksumAlgorithmCrc32,
|
||||
},
|
||||
{
|
||||
key: "obj-2/",
|
||||
expectedSum: "AAAAAA==",
|
||||
checksumAlgo: types.ChecksumAlgorithmCrc32c,
|
||||
},
|
||||
{
|
||||
key: "obj-3/",
|
||||
expectedSum: "AAAAAAAAAAA=",
|
||||
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
|
||||
},
|
||||
{
|
||||
key: "obj-4/",
|
||||
expectedSum: "2jmj7l5rSw0yVb/vlWAYkK/YBwk=",
|
||||
checksumAlgo: types.ChecksumAlgorithmSha1,
|
||||
},
|
||||
{
|
||||
key: "obj-5/",
|
||||
expectedSum: "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
|
||||
checksumAlgo: types.ChecksumAlgorithmSha256,
|
||||
},
|
||||
} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj.key,
|
||||
ChecksumAlgorithm: obj.checksumAlgo,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("test %v failed: %w", i+1, err)
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj.key,
|
||||
ChecksumMode: types.ChecksumModeEnabled,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("test %v failed: %w", i+1, err)
|
||||
}
|
||||
|
||||
if res.ChecksumType != types.ChecksumTypeFullObject {
|
||||
return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v",
|
||||
i+1, obj.key, types.ChecksumTypeFullObject, res.ChecksumType)
|
||||
}
|
||||
|
||||
var gotSum *string
|
||||
switch obj.checksumAlgo {
|
||||
case types.ChecksumAlgorithmCrc32:
|
||||
gotSum = res.ChecksumCRC32
|
||||
case types.ChecksumAlgorithmCrc32c:
|
||||
gotSum = res.ChecksumCRC32C
|
||||
case types.ChecksumAlgorithmCrc64nvme:
|
||||
gotSum = res.ChecksumCRC64NVME
|
||||
case types.ChecksumAlgorithmSha1:
|
||||
gotSum = res.ChecksumSHA1
|
||||
case types.ChecksumAlgorithmSha256:
|
||||
gotSum = res.ChecksumSHA256
|
||||
}
|
||||
|
||||
if getString(gotSum) != obj.expectedSum {
|
||||
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, obj.checksumAlgo, obj.expectedSum, getString(gotSum))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func GetObject_large_object(s *S3Conf) error {
|
||||
testName := "GetObject_large_object"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
|
||||
@@ -698,6 +698,7 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
|
||||
testName := "PutObject_incorrect_checksums"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
obj := "my-obj"
|
||||
dirObj := "dir-object/"
|
||||
|
||||
for i, el := range []struct {
|
||||
algo types.ChecksumAlgorithm
|
||||
@@ -728,6 +729,7 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
|
||||
crc64nvme: getPtr("sV264W+gYBI="),
|
||||
},
|
||||
} {
|
||||
// test for file object
|
||||
_, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj,
|
||||
@@ -740,6 +742,22 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
|
||||
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// test for directory object
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &dirObj,
|
||||
ChecksumCRC32: el.crc32,
|
||||
ChecksumCRC32C: el.crc32c,
|
||||
ChecksumSHA1: el.sha1,
|
||||
ChecksumSHA256: el.sha256,
|
||||
ChecksumCRC64NVME: el.crc64nvme,
|
||||
})
|
||||
cancel()
|
||||
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -786,6 +804,91 @@ func PutObject_default_checksum(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func PutObject_dir_object_default_checksum(s *S3Conf) error {
|
||||
testName := "PutObject_dir_object_default_checksum"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
obj := "dir/obj/"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj,
|
||||
}, func(o *s3.Options) {
|
||||
o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if res.ChecksumType != types.ChecksumTypeFullObject {
|
||||
return fmt.Errorf("expected the object checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType)
|
||||
}
|
||||
expectedcrc64nvme := "AAAAAAAAAAA="
|
||||
if getString(res.ChecksumCRC64NVME) != expectedcrc64nvme {
|
||||
return fmt.Errorf("expected the crc64nvme checksum to be %s, instead got %s", expectedcrc64nvme, getString(res.ChecksumCRC64NVME))
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PutObject_dir_object_checksums_success(s *S3Conf) error {
|
||||
testName := "PutObject_dir_object_checksums_success"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
for i, test := range []struct {
|
||||
algo types.ChecksumAlgorithm
|
||||
checksumValue string
|
||||
}{
|
||||
{types.ChecksumAlgorithmCrc32, "AAAAAA=="},
|
||||
{types.ChecksumAlgorithmCrc32c, "AAAAAA=="},
|
||||
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
|
||||
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
|
||||
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
|
||||
} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("obj-%v/", i)),
|
||||
ChecksumAlgorithm: test.algo,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if res.ChecksumType != types.ChecksumTypeFullObject {
|
||||
return fmt.Errorf("expected the checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType)
|
||||
}
|
||||
|
||||
switch test.algo {
|
||||
case types.ChecksumAlgorithmCrc32:
|
||||
if getString(res.ChecksumCRC32) != test.checksumValue {
|
||||
return fmt.Errorf("expected the crc32 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32))
|
||||
}
|
||||
case types.ChecksumAlgorithmCrc32c:
|
||||
if getString(res.ChecksumCRC32C) != test.checksumValue {
|
||||
return fmt.Errorf("expected the crc32c checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32C))
|
||||
}
|
||||
case types.ChecksumAlgorithmSha1:
|
||||
if getString(res.ChecksumSHA1) != test.checksumValue {
|
||||
return fmt.Errorf("expected the sha1 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA1))
|
||||
}
|
||||
case types.ChecksumAlgorithmSha256:
|
||||
if getString(res.ChecksumSHA256) != test.checksumValue {
|
||||
return fmt.Errorf("expected the sha256 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA256))
|
||||
}
|
||||
case types.ChecksumAlgorithmCrc64nvme:
|
||||
if getString(res.ChecksumCRC64NVME) != test.checksumValue {
|
||||
return fmt.Errorf("expected the crc64nvme checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC64NVME))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PutObject_checksums_success(s *S3Conf) error {
|
||||
testName := "PutObject_checksums_success"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
|
||||
@@ -178,7 +178,9 @@ func TestPutObject(ts *TestState) {
|
||||
ts.Run(PutObject_invalid_checksum_header)
|
||||
ts.Run(PutObject_incorrect_checksums)
|
||||
ts.Run(PutObject_default_checksum)
|
||||
ts.Run(PutObject_dir_object_default_checksum)
|
||||
ts.Run(PutObject_checksums_success)
|
||||
ts.Run(PutObject_dir_object_checksums_success)
|
||||
// azure applies some encoding mechanisms.
|
||||
ts.Run(PutObject_false_negative_object_names)
|
||||
// azure doesn't support these metadata characters
|
||||
@@ -239,6 +241,7 @@ func TestGetObject(ts *TestState) {
|
||||
//TODO: remove the condition after implementing checksums in azure
|
||||
if !ts.conf.azureTests {
|
||||
ts.Run(GetObject_checksums)
|
||||
ts.Run(GetObject_dir_object_checksum)
|
||||
}
|
||||
ts.Run(GetObject_success)
|
||||
ts.Run(GetObject_directory_success)
|
||||
@@ -1247,7 +1250,9 @@ func GetIntTests() IntTests {
|
||||
"PutObject_invalid_checksum_header": PutObject_invalid_checksum_header,
|
||||
"PutObject_incorrect_checksums": PutObject_incorrect_checksums,
|
||||
"PutObject_default_checksum": PutObject_default_checksum,
|
||||
"PutObject_dir_object_default_checksum": PutObject_dir_object_default_checksum,
|
||||
"PutObject_checksums_success": PutObject_checksums_success,
|
||||
"PutObject_dir_object_checksums_success": PutObject_dir_object_checksums_success,
|
||||
"PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data,
|
||||
"PresignedAuth_Put_GetObject_with_UTF8_chars": PresignedAuth_Put_GetObject_with_UTF8_chars,
|
||||
"PresignedAuth_UploadPart": PresignedAuth_UploadPart,
|
||||
@@ -1346,6 +1351,7 @@ func GetIntTests() IntTests {
|
||||
"GetObject_large_object": GetObject_large_object,
|
||||
"GetObject_conditional_reads": GetObject_conditional_reads,
|
||||
"GetObject_checksums": GetObject_checksums,
|
||||
"GetObject_dir_object_checksum": GetObject_dir_object_checksum,
|
||||
"GetObject_success": GetObject_success,
|
||||
"GetObject_directory_success": GetObject_directory_success,
|
||||
"GetObject_by_range_resp_status": GetObject_by_range_resp_status,
|
||||
|
||||
Reference in New Issue
Block a user