mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-08 09:06:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: makes object metadata keys lowercase in object creation actions

Browse Source 
Fixes #1482

The metadata keys should always be converted to lowercase in `PutObject`, `CreateMultipartUpload`, and `CopyObject`. This implementation converts the metadata keys to lowercase in the front end, ensuring they are stored in lowercase in the backend.
This commit is contained in:
niksis02
2025-10-28 21:47:13 +04:00
parent d63b5818f1
commit 045bdec60c
4 changed files with 250 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3api/utils/utils.go
+1 -1
View File
@@ -58,7 +58,7 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin
for key, value := range headers.AllInOrder() {
hKey := string(key)
if strings.HasPrefix(strings.ToLower(hKey), "x-amz-meta-") {
trimmedKey := hKey[11:]
trimmedKey := strings.ToLower(hKey[11:])
headerValue := string(value)
metadata[trimmedKey] = headerValue
}
tests/integration/group-tests.go
+9
View File
@@ -169,6 +169,8 @@ func TestPutObject(ts *TestState) {
ts.Run(PutObject_checksums_success)
// azure applies some encoding mechanisms.
ts.Run(PutObject_false_negative_object_names)
// azure doesn't support these metadata characters
ts.Run(PutObject_with_metadata)
}
ts.Run(PutObject_success)
if !ts.conf.versioningEnabled {
@@ -329,6 +331,8 @@ func TestCopyObject(ts *TestState) {
ts.Run(CopyObject_should_copy_the_existing_checksum)
ts.Run(CopyObject_should_replace_the_existing_checksum)
ts.Run(CopyObject_to_itself_by_replacing_the_checksum)
// azure doesn't support these metadata characters
ts.Run(CopyObject_with_metadata)
}
ts.Run(CopyObject_success)
}
@@ -487,6 +491,8 @@ func TestCompleteMultipartUpload(ts *TestState) {
ts.Run(CompleteMultipartUpload_checksum_type_mismatch)
ts.Run(CompleteMultipartUpload_should_ignore_the_final_checksum)
ts.Run(CompleteMultipartUpload_should_succeed_without_final_checksum_type)
// azure doesn't support these metadata characters
ts.Run(CompleteMultipartUpload_with_metadata)
}
ts.Run(CompleteMultipartUpload_success)
if !ts.conf.azureTests {
@@ -1107,6 +1113,7 @@ func GetIntTests() IntTests {
"PutObject_invalid_legal_hold": PutObject_invalid_legal_hold,
"PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode,
"PutObject_conditional_writes": PutObject_conditional_writes,
"PutObject_with_metadata": PutObject_with_metadata,
"PutObject_invalid_credentials": PutObject_invalid_credentials,
"PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch,
"PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers,
@@ -1272,6 +1279,7 @@ func GetIntTests() IntTests {
"CopyObject_with_legal_hold": CopyObject_with_legal_hold,
"CopyObject_with_retention_lock": CopyObject_with_retention_lock,
"CopyObject_conditional_reads": CopyObject_conditional_reads,
"CopyObject_with_metadata": CopyObject_with_metadata,
"CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm,
"CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy,
"CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum,
@@ -1369,6 +1377,7 @@ func GetIntTests() IntTests {
"CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order,
"CompleteMultipartUpload_mpu_object_size": CompleteMultipartUpload_mpu_object_size,
"CompleteMultipartUpload_conditional_writes": CompleteMultipartUpload_conditional_writes,
"CompleteMultipartUpload_with_metadata": CompleteMultipartUpload_with_metadata,
"CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type,
"CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part,
"CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part,
tests/integration/tests.go
+234 -1
View File
@@ -3226,6 +3226,69 @@ func PutObject_conditional_writes(s *S3Conf) error {
})
}
func PutObject_with_metadata(s *S3Conf) error {
testName := "PutObject_with_metadata"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
meta := map[string]string{
"Key": "Val",
"X-Test": "Example",
"UPPERCASE": "should-remain",
"MiXeD-CaSe": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"KeyURL": "https://example.com/test?query=1",
"EmptyValue": "",
"LongKeyNameThatShouldStillBeValidButQuiteLongToTestLimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"WhitespaceKey ": " trailing-key",
}
obj := "my-object"
_, err := putObjectWithData(3, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Metadata: meta,
}, s3client)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
expectedMeta := map[string]string{
"key": "Val",
"x-test": "Example",
"uppercase": "should-remain",
"mixed-case": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"keyurl": "https://example.com/test?query=1",
"emptyvalue": "",
"longkeynamethatshouldstillbevalidbutquitelongtotestlimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"whitespacekey": "trailing-key",
}
if !areMapsSame(expectedMeta, res.Metadata) {
return fmt.Errorf("expected the object metadata to be %v, instead got %v", expectedMeta, res.Metadata)
}
return nil
})
}
func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error {
testName := "PutObject_checksum_algorithm_and_header_mismatch"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -7780,6 +7843,10 @@ func CopyObject_to_itself_with_new_metadata(s *S3Conf) error {
return err
}
meta = map[string]string{
"hello": "World",
}
if !areMapsSame(resp.Metadata, meta) {
return fmt.Errorf("expected uploaded object metadata to be %v, instead got %v",
meta, resp.Metadata)
@@ -7787,7 +7854,7 @@ func CopyObject_to_itself_with_new_metadata(s *S3Conf) error {
// verify updating metadata has correct meta
meta = map[string]string{
"New": "Metadata",
"new": "Metadata",
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
@@ -8435,6 +8502,85 @@ func CopyObject_conditional_reads(s *S3Conf) error {
})
}
func CopyObject_with_metadata(s *S3Conf) error {
testName := "CopyObject_with_metadata"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
srcObj, dstObj := "src-obj", "dst-obj"
_, err := putObjectWithData(2, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
Metadata: map[string]string{
"key": "value",
},
}, s3client)
if err != nil {
return err
}
meta := map[string]string{
"Key": "Val",
"X-Test": "Example",
"UPPERCASE": "should-remain",
"MiXeD-CaSe": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"KeyURL": "https://example.com/test?query=1",
"EmptyValue": "",
"LongKeyNameThatShouldStillBeValidButQuiteLongToTestLimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"WhitespaceKey ": " trailing-key",
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: &bucket,
Key: &dstObj,
Metadata: meta,
CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, srcObj)),
MetadataDirective: types.MetadataDirectiveReplace,
})
cancel()
if err != nil {
return err
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &dstObj,
})
cancel()
if err != nil {
return err
}
expectedMeta := map[string]string{
"key": "Val",
"x-test": "Example",
"uppercase": "should-remain",
"mixed-case": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"keyurl": "https://example.com/test?query=1",
"emptyvalue": "",
"longkeynamethatshouldstillbevalidbutquitelongtotestlimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"whitespacekey": "trailing-key",
}
if !areMapsSame(expectedMeta, res.Metadata) {
return fmt.Errorf("expected the object metadata to be %v, instead got %v", expectedMeta, res.Metadata)
}
return nil
})
}
func CopyObject_invalid_checksum_algorithm(s *S3Conf) error {
testName := "CopyObject_invalid_checksum_algorithm"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -13645,6 +13791,93 @@ func CompleteMultipartUpload_conditional_writes(s *S3Conf) error {
})
}
func CompleteMultipartUpload_with_metadata(s *S3Conf) error {
testName := "CompleteMultipartUpload_with_metadata"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
meta := map[string]string{
"Key": "Val",
"X-Test": "Example",
"UPPERCASE": "should-remain",
"MiXeD-CaSe": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"KeyURL": "https://example.com/test?query=1",
"EmptyValue": "",
"LongKeyNameThatShouldStillBeValidButQuiteLongToTestLimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"WhitespaceKey ": " trailing-key",
}
obj := "my-object"
mp, err := createMp(s3client, bucket, obj, withMetadata(meta))
if err != nil {
return err
}
parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, obj, *mp.UploadId)
if err != nil {
return err
}
compParts := []types.CompletedPart{}
for _, el := range parts {
compParts = append(compParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
})
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: compParts,
},
})
cancel()
if err != nil {
return err
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
expectedMeta := map[string]string{
"key": "Val",
"x-test": "Example",
"uppercase": "should-remain",
"mixed-case": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"keyurl": "https://example.com/test?query=1",
"emptyvalue": "",
"longkeynamethatshouldstillbevalidbutquitelongtotestlimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"whitespacekey": "trailing-key",
}
if !areMapsSame(expectedMeta, res.Metadata) {
return fmt.Errorf("expected the object metadata to be %v, instead got %v", expectedMeta, res.Metadata)
}
return nil
})
}
func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error {
testName := "CompleteMultipartUpload_invalid_part_number"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
tests/integration/utils.go
+6 -1
View File
@@ -535,6 +535,7 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client)
type mpCfg struct {
checksumAlgorithm types.ChecksumAlgorithm
checksumType types.ChecksumType
metadata map[string]string
}
type mpOpt func(*mpCfg)
@@ -545,6 +546,9 @@ func withChecksum(algo types.ChecksumAlgorithm) mpOpt {
func withChecksumType(t types.ChecksumType) mpOpt {
return func(mc *mpCfg) { mc.checksumType = t }
}
func withMetadata(m map[string]string) mpOpt {
return func(mc *mpCfg) { mc.metadata = m }
}
func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) {
cfg := new(mpCfg)
@@ -557,6 +561,7 @@ func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.Creat
Key: &key,
ChecksumAlgorithm: cfg.checksumAlgorithm,
ChecksumType: cfg.checksumType,
Metadata: cfg.metadata,
})
cancel()
return out, err
@@ -710,7 +715,7 @@ func areMapsSame(mp1, mp2 map[string]string) bool {
return false
}
for key, val := range mp2 {
if mp1[strings.ToLower(key)] != val {
if mp1[key] != val {
return false
}
}