From 045bdec60cc7accc71dbed1c6a019cda7b2bfbd9 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Tue, 28 Oct 2025 21:47:13 +0400 Subject: [PATCH] fix: makes object metadata keys lowercase in object creation actions Fixes #1482 The metadata keys should always be converted to lowercase in `PutObject`, `CreateMultipartUpload`, and `CopyObject`. This implementation converts the metadata keys to lowercase in the front end, ensuring they are stored in lowercase in the backend. --- s3api/utils/utils.go | 2 +- tests/integration/group-tests.go | 9 ++ tests/integration/tests.go | 235 ++++++++++++++++++++++++++++++- tests/integration/utils.go | 7 +- 4 files changed, 250 insertions(+), 3 deletions(-) diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 3e0022e..dc0904c 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -58,7 +58,7 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin for key, value := range headers.AllInOrder() { hKey := string(key) if strings.HasPrefix(strings.ToLower(hKey), "x-amz-meta-") { - trimmedKey := hKey[11:] + trimmedKey := strings.ToLower(hKey[11:]) headerValue := string(value) metadata[trimmedKey] = headerValue } diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index bfd6ee6..ec209d9 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -169,6 +169,8 @@ func TestPutObject(ts *TestState) { ts.Run(PutObject_checksums_success) // azure applies some encoding mechanisms. ts.Run(PutObject_false_negative_object_names) + // azure doesn't support these metadata characters + ts.Run(PutObject_with_metadata) } ts.Run(PutObject_success) if !ts.conf.versioningEnabled { @@ -329,6 +331,8 @@ func TestCopyObject(ts *TestState) { ts.Run(CopyObject_should_copy_the_existing_checksum) ts.Run(CopyObject_should_replace_the_existing_checksum) ts.Run(CopyObject_to_itself_by_replacing_the_checksum) + // azure doesn't support these metadata characters + ts.Run(CopyObject_with_metadata) } ts.Run(CopyObject_success) } @@ -487,6 +491,8 @@ func TestCompleteMultipartUpload(ts *TestState) { ts.Run(CompleteMultipartUpload_checksum_type_mismatch) ts.Run(CompleteMultipartUpload_should_ignore_the_final_checksum) ts.Run(CompleteMultipartUpload_should_succeed_without_final_checksum_type) + // azure doesn't support these metadata characters + ts.Run(CompleteMultipartUpload_with_metadata) } ts.Run(CompleteMultipartUpload_success) if !ts.conf.azureTests { @@ -1107,6 +1113,7 @@ func GetIntTests() IntTests { "PutObject_invalid_legal_hold": PutObject_invalid_legal_hold, "PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode, "PutObject_conditional_writes": PutObject_conditional_writes, + "PutObject_with_metadata": PutObject_with_metadata, "PutObject_invalid_credentials": PutObject_invalid_credentials, "PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch, "PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers, @@ -1272,6 +1279,7 @@ func GetIntTests() IntTests { "CopyObject_with_legal_hold": CopyObject_with_legal_hold, "CopyObject_with_retention_lock": CopyObject_with_retention_lock, "CopyObject_conditional_reads": CopyObject_conditional_reads, + "CopyObject_with_metadata": CopyObject_with_metadata, "CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm, "CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy, "CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum, @@ -1369,6 +1377,7 @@ func GetIntTests() IntTests { "CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order, "CompleteMultipartUpload_mpu_object_size": CompleteMultipartUpload_mpu_object_size, "CompleteMultipartUpload_conditional_writes": CompleteMultipartUpload_conditional_writes, + "CompleteMultipartUpload_with_metadata": CompleteMultipartUpload_with_metadata, "CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type, "CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part, "CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index dca5ebf..5431257 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -3226,6 +3226,69 @@ func PutObject_conditional_writes(s *S3Conf) error { }) } +func PutObject_with_metadata(s *S3Conf) error { + testName := "PutObject_with_metadata" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + meta := map[string]string{ + "Key": "Val", + "X-Test": "Example", + "UPPERCASE": "should-remain", + "MiXeD-CaSe": "normalize-to-lower", + "with-number-123": "numeric-test", + "123numeric-prefix": "value123", + "key_with_underscore": "underscore-ok", + "key-with-dash": "dash-ok", + "key.with.dot": "dot-ok", + "KeyURL": "https://example.com/test?query=1", + "EmptyValue": "", + "LongKeyNameThatShouldStillBeValidButQuiteLongToTestLimits": "some long metadata value to ensure nothing breaks at higher header sizes", + "WhitespaceKey ": " trailing-key", + } + + obj := "my-object" + _, err := putObjectWithData(3, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: meta, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + expectedMeta := map[string]string{ + "key": "Val", + "x-test": "Example", + "uppercase": "should-remain", + "mixed-case": "normalize-to-lower", + "with-number-123": "numeric-test", + "123numeric-prefix": "value123", + "key_with_underscore": "underscore-ok", + "key-with-dash": "dash-ok", + "key.with.dot": "dot-ok", + "keyurl": "https://example.com/test?query=1", + "emptyvalue": "", + "longkeynamethatshouldstillbevalidbutquitelongtotestlimits": "some long metadata value to ensure nothing breaks at higher header sizes", + "whitespacekey": "trailing-key", + } + + if !areMapsSame(expectedMeta, res.Metadata) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", expectedMeta, res.Metadata) + } + + return nil + }) +} + func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error { testName := "PutObject_checksum_algorithm_and_header_mismatch" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -7780,6 +7843,10 @@ func CopyObject_to_itself_with_new_metadata(s *S3Conf) error { return err } + meta = map[string]string{ + "hello": "World", + } + if !areMapsSame(resp.Metadata, meta) { return fmt.Errorf("expected uploaded object metadata to be %v, instead got %v", meta, resp.Metadata) @@ -7787,7 +7854,7 @@ func CopyObject_to_itself_with_new_metadata(s *S3Conf) error { // verify updating metadata has correct meta meta = map[string]string{ - "New": "Metadata", + "new": "Metadata", } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ @@ -8435,6 +8502,85 @@ func CopyObject_conditional_reads(s *S3Conf) error { }) } +func CopyObject_with_metadata(s *S3Conf) error { + testName := "CopyObject_with_metadata" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj, dstObj := "src-obj", "dst-obj" + + _, err := putObjectWithData(2, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + Metadata: map[string]string{ + "key": "value", + }, + }, s3client) + if err != nil { + return err + } + + meta := map[string]string{ + "Key": "Val", + "X-Test": "Example", + "UPPERCASE": "should-remain", + "MiXeD-CaSe": "normalize-to-lower", + "with-number-123": "numeric-test", + "123numeric-prefix": "value123", + "key_with_underscore": "underscore-ok", + "key-with-dash": "dash-ok", + "key.with.dot": "dot-ok", + "KeyURL": "https://example.com/test?query=1", + "EmptyValue": "", + "LongKeyNameThatShouldStillBeValidButQuiteLongToTestLimits": "some long metadata value to ensure nothing breaks at higher header sizes", + "WhitespaceKey ": " trailing-key", + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + Metadata: meta, + CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, srcObj)), + MetadataDirective: types.MetadataDirectiveReplace, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + }) + cancel() + if err != nil { + return err + } + + expectedMeta := map[string]string{ + "key": "Val", + "x-test": "Example", + "uppercase": "should-remain", + "mixed-case": "normalize-to-lower", + "with-number-123": "numeric-test", + "123numeric-prefix": "value123", + "key_with_underscore": "underscore-ok", + "key-with-dash": "dash-ok", + "key.with.dot": "dot-ok", + "keyurl": "https://example.com/test?query=1", + "emptyvalue": "", + "longkeynamethatshouldstillbevalidbutquitelongtotestlimits": "some long metadata value to ensure nothing breaks at higher header sizes", + "whitespacekey": "trailing-key", + } + + if !areMapsSame(expectedMeta, res.Metadata) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", expectedMeta, res.Metadata) + } + + return nil + }) +} + func CopyObject_invalid_checksum_algorithm(s *S3Conf) error { testName := "CopyObject_invalid_checksum_algorithm" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -13645,6 +13791,93 @@ func CompleteMultipartUpload_conditional_writes(s *S3Conf) error { }) } +func CompleteMultipartUpload_with_metadata(s *S3Conf) error { + testName := "CompleteMultipartUpload_with_metadata" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + meta := map[string]string{ + "Key": "Val", + "X-Test": "Example", + "UPPERCASE": "should-remain", + "MiXeD-CaSe": "normalize-to-lower", + "with-number-123": "numeric-test", + "123numeric-prefix": "value123", + "key_with_underscore": "underscore-ok", + "key-with-dash": "dash-ok", + "key.with.dot": "dot-ok", + "KeyURL": "https://example.com/test?query=1", + "EmptyValue": "", + "LongKeyNameThatShouldStillBeValidButQuiteLongToTestLimits": "some long metadata value to ensure nothing breaks at higher header sizes", + "WhitespaceKey ": " trailing-key", + } + + obj := "my-object" + + mp, err := createMp(s3client, bucket, obj, withMetadata(meta)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + compParts := []types.CompletedPart{} + for _, el := range parts { + compParts = append(compParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + expectedMeta := map[string]string{ + "key": "Val", + "x-test": "Example", + "uppercase": "should-remain", + "mixed-case": "normalize-to-lower", + "with-number-123": "numeric-test", + "123numeric-prefix": "value123", + "key_with_underscore": "underscore-ok", + "key-with-dash": "dash-ok", + "key.with.dot": "dot-ok", + "keyurl": "https://example.com/test?query=1", + "emptyvalue": "", + "longkeynamethatshouldstillbevalidbutquitelongtotestlimits": "some long metadata value to ensure nothing breaks at higher header sizes", + "whitespacekey": "trailing-key", + } + + if !areMapsSame(expectedMeta, res.Metadata) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", expectedMeta, res.Metadata) + } + + return nil + }) +} + func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error { testName := "CompleteMultipartUpload_invalid_part_number" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 807c7c7..ba8b8ca 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -535,6 +535,7 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) type mpCfg struct { checksumAlgorithm types.ChecksumAlgorithm checksumType types.ChecksumType + metadata map[string]string } type mpOpt func(*mpCfg) @@ -545,6 +546,9 @@ func withChecksum(algo types.ChecksumAlgorithm) mpOpt { func withChecksumType(t types.ChecksumType) mpOpt { return func(mc *mpCfg) { mc.checksumType = t } } +func withMetadata(m map[string]string) mpOpt { + return func(mc *mpCfg) { mc.metadata = m } +} func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) { cfg := new(mpCfg) @@ -557,6 +561,7 @@ func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.Creat Key: &key, ChecksumAlgorithm: cfg.checksumAlgorithm, ChecksumType: cfg.checksumType, + Metadata: cfg.metadata, }) cancel() return out, err @@ -710,7 +715,7 @@ func areMapsSame(mp1, mp2 map[string]string) bool { return false } for key, val := range mp2 { - if mp1[strings.ToLower(key)] != val { + if mp1[key] != val { return false } }