mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-19 05:36:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: change the way object metadata is stored in posix

Browse Source 
Fixes #1909

Previously, the mapping between object metadata and posix object was as follows: for each metadata key, we stored a separate xattr with the `user.X-Amz-Meta.<key>` prefix. This resulted in syscall overhead when storing and deleting large numbers of metadata keys.
In addition, very long metadata keys caused failures because most posix filesystems limit xattr key lengths to 127–255 bytes, while S3 does not enforce such a per-key limit.

The logic has now been changed so that all object metadata is stored in a single xattr, `user.metadata`, as a JSON key/value object. For backward compatibility, metadata GET operations still fall back to the old mechanism (`metadata key -> xattr key`) when `user.metadata` is not present.

A new CLI utility has been added to convert all legacy object metadata to the new metadata format within the provided directory.

**Example usage:**

```
versitygw utils convert-xattr-metadata path/to/bucket
```

or

```
versitygw utils cxm path/to/bucket
```

It is recommended to run this command on bucket directories to convert all legacy metadata for every object in the bucket.
This commit is contained in:
niksis02
2026-03-06 01:44:14 +04:00
parent b581a4c169
commit 97bb70509f
5 changed files with 340 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/posix/posix.go
+142 -109
View File
@@ -101,7 +101,8 @@ const (
MetaTmpMultipartDir = MetaTmpDir + "/multipart"
onameAttr = "objname"
tagHdr = "X-Amz-Tagging"
metaHdr = "X-Amz-Meta"
oldMetaHdr = "X-Amz-Meta"
metadataHdr = "metadata"
contentTypeHdr = "content-type"
contentEncHdr = "content-encoding"
contentLangHdr = "content-language"
@@ -1471,18 +1472,6 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create
return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("set name attr for upload: %w", err)
}
// set user metadata
for k, v := range mpu.Metadata {
err := p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID),
fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
if err != nil {
// cleanup object if returning error
os.RemoveAll(filepath.Join(tmppath, uploadID))
os.Remove(tmppath)
return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("set user attr %q: %w", k, err)
}
}
// set object tagging
if tags != nil {
err := p.PutObjectTagging(withCtxNoSlot(ctx), bucket, filepath.Join(objdir, uploadID), "", tags)
@@ -1494,14 +1483,16 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create
}
}
err = p.storeObjectMetadata(nil, bucket, filepath.Join(objdir, uploadID), objectMetadata{
ContentType: mpu.ContentType,
ContentEncoding: mpu.ContentEncoding,
ContentDisposition: mpu.ContentDisposition,
ContentLanguage: mpu.ContentLanguage,
CacheControl: mpu.CacheControl,
Expires: mpu.Expires,
})
err = p.storeObjectMetaProperties(nil, bucket, filepath.Join(objdir, uploadID),
metaProperties{
ContentType: mpu.ContentType,
ContentEncoding: mpu.ContentEncoding,
ContentDisposition: mpu.ContentDisposition,
ContentLanguage: mpu.ContentLanguage,
CacheControl: mpu.CacheControl,
Expires: mpu.Expires,
Metadata: mpu.Metadata,
})
if err != nil {
// cleanup object if returning error
os.RemoveAll(filepath.Join(tmppath, uploadID))
@@ -1840,9 +1831,8 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
upiddir := filepath.Join(objdir, uploadID)
userMetaData := make(map[string]string)
objMeta := p.loadObjectMetaData(nil, bucket, upiddir, nil, userMetaData)
err = p.storeObjectMetadata(f.File(), bucket, object, objMeta)
objMeta := p.loadObjectMetaProperties(nil, bucket, upiddir, nil)
err = p.storeObjectMetaProperties(f.File(), bucket, object, objMeta)
if err != nil {
return res, "", err
}
@@ -1884,13 +1874,6 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
}
}
for k, v := range userMetaData {
err = p.meta.StoreAttribute(f.File(), bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
if err != nil {
return res, "", fmt.Errorf("set user attr %q: %w", k, err)
}
}
// load and set tagging
tagging, err := p.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -2109,41 +2092,74 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte,
return sum, nil
}
type objectMetadata struct {
type metaProperties struct {
ContentType *string
ContentEncoding *string
ContentDisposition *string
ContentLanguage *string
CacheControl *string
Expires *string
Metadata map[string]string
}
// fill out the user metadata map with the metadata for the object
// and return object meta properties as `ObjectMetadata`
func (p *Posix) loadObjectMetaData(f *os.File, bucket, object string, fi *os.FileInfo, m map[string]string) objectMetadata {
ents, err := p.meta.ListAttributes(bucket, object)
if err != nil || len(ents) == 0 {
return objectMetadata{}
}
// loadObjectMetadata loads the given object metadata, if it fails to load
// it falls back to the old metadata key -> xattr key mapping mechanism
// and converts the old metadata storing schema with the new one
func (p *Posix) loadObjectMetadata(f *os.File, bucket, object string) map[string]string {
m, err := p.meta.RetrieveAttribute(f, bucket, object, metadataHdr)
if err != nil {
if errors.Is(err, meta.ErrNoSuchKey) {
// fallback to the deprecated mechanism
ents, err := p.meta.ListAttributes(bucket, object)
if err != nil || len(ents) == 0 {
return nil
}
result := map[string]string{}
if m != nil {
for _, e := range ents {
if !isValidMeta(e) {
continue
legacyAttrs := make([]string, 0, len(ents))
for _, e := range ents {
if !isValidMeta(e) {
continue
}
b, err := p.meta.RetrieveAttribute(f, bucket, object, e)
if err != nil {
continue
}
if b == nil {
b = []byte{}
}
legacyAttrs = append(legacyAttrs, e)
result[strings.TrimPrefix(e, fmt.Sprintf("%v.", oldMetaHdr))] = string(b)
}
b, err := p.meta.RetrieveAttribute(f, bucket, object, e)
if err != nil {
continue
if len(result) != 0 {
err = p.storeObjectMetadata(f, bucket, object, result)
if err == nil {
// if it succeded to store the metadata as a json
// object in user.metadata, cleanup the legacy X-Amz-Meta.
// attributes
for _, attr := range legacyAttrs {
_ = p.meta.DeleteAttribute(bucket, object, attr)
}
}
}
if b == nil {
m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = ""
continue
}
m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b)
return result
}
return nil
}
var result objectMetadata
var metadata map[string]string
_ = json.Unmarshal(m, &metadata)
return metadata
}
// loadObjectMetaProperties loads the given object meta properties
// including Content-Type, Cache-Control ... and the object Metadata
func (p *Posix) loadObjectMetaProperties(f *os.File, bucket, object string, fi *os.FileInfo) metaProperties {
var result metaProperties
b, err := p.meta.RetrieveAttribute(f, bucket, object, contentTypeHdr)
if err == nil {
@@ -2182,10 +2198,49 @@ func (p *Posix) loadObjectMetaData(f *os.File, bucket, object string, fi *os.Fil
result.Expires = backend.GetPtrFromString(string(b))
}
result.Metadata = p.loadObjectMetadata(f, bucket, object)
return result
}
func (p *Posix) storeObjectMetadata(f *os.File, bucket, object string, m objectMetadata) error {
// storeObjectMetadata parses the given metadata map[string]string
// to raw json to store in file/directory meta storage and checks/cleans up
// any legacy metadata persent in the object meta storage
func (p *Posix) storeObjectMetadata(f *os.File, bucket, object string, meta map[string]string) error {
if len(meta) == 0 {
return nil
}
parsed, err := json.Marshal(meta)
if err != nil {
return fmt.Errorf("marshal metadata: %w", err)
}
err = p.meta.StoreAttribute(f, bucket, object, metadataHdr, parsed)
if err != nil {
return err
}
// cleanup any previously set legacy metadata
ents, err := p.meta.ListAttributes(bucket, object)
if err != nil || len(ents) == 0 {
return nil
}
for _, ent := range ents {
if !isValidMeta(ent) {
continue
}
_ = p.meta.DeleteAttribute(bucket, object, ent)
}
return nil
}
// storeObjectMetaProperties stores the object meta properties like:
// Content-Type, Content-Encoding, object Metadata ...
func (p *Posix) storeObjectMetaProperties(f *os.File, bucket, object string, m metaProperties) error {
if getString(m.ContentType) != "" {
err := p.meta.StoreAttribute(f, bucket, object, contentTypeHdr, []byte(*m.ContentType))
if err != nil {
@@ -2222,12 +2277,18 @@ func (p *Posix) storeObjectMetadata(f *os.File, bucket, object string, m objectM
return fmt.Errorf("set expires: %w", err)
}
}
if m.Metadata != nil {
err := p.storeObjectMetadata(f, bucket, object, m.Metadata)
if err != nil {
return fmt.Errorf("set metadata: %w", err)
}
}
return nil
}
func isValidMeta(val string) bool {
return strings.HasPrefix(val, metaHdr)
return strings.HasPrefix(val, oldMetaHdr)
}
func (p *Posix) AbortMultipartUpload(ctx context.Context, mpu *s3.AbortMultipartUploadInput) error {
@@ -3227,13 +3288,9 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
return s3response.PutObjectOutput{}, err
}
// set object metadata
for k, v := range po.Metadata {
err := p.meta.StoreAttribute(nil, *po.Bucket, *po.Key,
fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("set user attr %q: %w", k, err)
}
err = p.storeObjectMetadata(nil, *po.Bucket, *po.Key, po.Metadata)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("set object metadata: %w", err)
}
// Set object tagging
@@ -3407,14 +3464,6 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
versionID = nullVersionId
}
for k, v := range po.Metadata {
err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key,
fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("set user attr %q: %w", k, err)
}
}
var sum string
if isTrailingChecksum {
@@ -3451,14 +3500,16 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err)
}
err = p.storeObjectMetadata(f.File(), *po.Bucket, *po.Key, objectMetadata{
ContentType: po.ContentType,
ContentEncoding: po.ContentEncoding,
ContentLanguage: po.ContentLanguage,
ContentDisposition: po.ContentDisposition,
CacheControl: po.CacheControl,
Expires: po.Expires,
})
err = p.storeObjectMetaProperties(f.File(), *po.Bucket, *po.Key,
metaProperties{
ContentType: po.ContentType,
ContentEncoding: po.ContentEncoding,
ContentLanguage: po.ContentLanguage,
ContentDisposition: po.ContentDisposition,
CacheControl: po.CacheControl,
Expires: po.Expires,
Metadata: po.Metadata,
})
if err != nil {
return s3response.PutObjectOutput{}, err
}
@@ -4089,9 +4140,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
return nil, err
}
userMetaData := make(map[string]string)
objMeta := p.loadObjectMetaData(nil, bucket, object, &fid, userMetaData)
objMeta := p.loadObjectMetaProperties(nil, bucket, object, &fid)
var tagCount *int32
tags, err := p.getAttrTags(bucket, object, versionId)
@@ -4129,7 +4178,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
ExpiresString: objMeta.Expires,
ETag: &etag,
LastModified: backend.GetTimePtr(fid.ModTime()),
Metadata: userMetaData,
Metadata: objMeta.Metadata,
TagCount: tagCount,
ContentRange: nil,
StorageClass: types.StorageClassStandard,
@@ -4183,9 +4232,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
startOffset, startOffset+length-1, objSize)
}
userMetaData := make(map[string]string)
objMeta := p.loadObjectMetaData(f, bucket, object, &fi, userMetaData)
objMeta := p.loadObjectMetaProperties(f, bucket, object, &fi)
var tagCount *int32
tags, err := p.getAttrTags(bucket, object, versionId)
@@ -4224,7 +4271,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
ExpiresString: objMeta.Expires,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
Metadata: objMeta.Metadata,
TagCount: tagCount,
ContentRange: &contentRange,
StorageClass: types.StorageClassStandard,
@@ -4345,8 +4392,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
versionId = string(vId)
}
userMetaData := make(map[string]string)
objMeta := p.loadObjectMetaData(nil, bucket, object, &fi, userMetaData)
objMeta := p.loadObjectMetaProperties(nil, bucket, object, &fi)
b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
etag := string(b)
@@ -4432,7 +4478,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
ExpiresString: objMeta.Expires,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
Metadata: objMeta.Metadata,
ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
ObjectLockMode: objectLockMode,
ObjectLockRetainUntilDate: objectLockRetainUntilDate,
@@ -4604,9 +4650,6 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
return s3response.CopyObjectOutput{}, err
}
mdmap := make(map[string]string)
p.loadObjectMetaData(nil, srcBucket, srcObject, &fi, mdmap)
var etag string
var version *string
var crc32 *string
@@ -4623,20 +4666,9 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
}
// Delete the object metadata
for k := range mdmap {
err := p.meta.DeleteAttribute(dstBucket, dstObject,
fmt.Sprintf("%v.%v", metaHdr, k))
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return s3response.CopyObjectOutput{}, fmt.Errorf("delete user metadata: %w", err)
}
}
// Store the new metadata
for k, v := range input.Metadata {
err := p.meta.StoreAttribute(nil, dstBucket, dstObject,
fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
if err != nil {
return s3response.CopyObjectOutput{}, fmt.Errorf("set user attr %q: %w", k, err)
}
err = p.meta.DeleteAttribute(dstBucket, dstObject, metadataHdr)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return s3response.CopyObjectOutput{}, fmt.Errorf("delete object metadata: %w", err)
}
checksums, err := p.retrieveChecksums(nil, dstBucket, dstObject)
@@ -4707,14 +4739,15 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
version = backend.GetPtrFromString(string(vId))
// Store the provided object meta properties
err = p.storeObjectMetadata(nil, dstBucket, dstObject,
objectMetadata{
err = p.storeObjectMetaProperties(nil, dstBucket, dstObject,
metaProperties{
ContentType: input.ContentType,
ContentEncoding: input.ContentEncoding,
ContentLanguage: input.ContentLanguage,
ContentDisposition: input.ContentDisposition,
CacheControl: input.CacheControl,
Expires: input.Expires,
Metadata: input.Metadata,
})
if err != nil {
return s3response.CopyObjectOutput{}, err
@@ -4765,14 +4798,14 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
// load and pass the source object meta properties, if metadata directive is "COPY"
if input.MetadataDirective != types.MetadataDirectiveReplace {
metaProps := p.loadObjectMetaData(nil, srcBucket, srcObject, &fi, nil)
metaProps := p.loadObjectMetaProperties(nil, srcBucket, srcObject, &fi)
putObjectInput.ContentEncoding = metaProps.ContentEncoding
putObjectInput.ContentDisposition = metaProps.ContentDisposition
putObjectInput.ContentLanguage = metaProps.ContentLanguage
putObjectInput.ContentType = metaProps.ContentType
putObjectInput.CacheControl = metaProps.CacheControl
putObjectInput.Expires = metaProps.Expires
putObjectInput.Metadata = mdmap
putObjectInput.Metadata = metaProps.Metadata
}
// pass the input tagging to PutObject, if tagging directive is "REPLACE"
cmd/versitygw/utils.go
+135
View File
@@ -16,11 +16,14 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/s3event"
)
@@ -42,6 +45,12 @@ func utilsCommand() *cli.Command {
},
},
},
{
Name: "convert-xattr-metadata",
Aliases: []string{"cxm"},
Usage: "Convert legacy X-Amz-Meta.* xattrs into user.metadata JSON and remove legacy keys.",
Action: convertXattrMetadata,
},
},
}
}
@@ -89,3 +98,129 @@ func generateEventFiltersConfig(ctx *cli.Context) error {
return nil
}
const (
newMetadataAttr = "metadata" // stored as user.metadata
oldMetadataHdr = "X-Amz-Meta" // legacy prefix
)
func convertXattrMetadata(ctx *cli.Context) error {
root := strings.TrimSpace(ctx.Args().First())
if root == "" {
return cli.Exit("missing directory: should be provided as command argument", 2)
}
absRoot, err := filepath.Abs(root)
if err != nil {
return fmt.Errorf("resolve directory: %w", err)
}
info, err := os.Stat(absRoot)
if err != nil {
return fmt.Errorf("stat directory: %w", err)
}
if !info.IsDir() {
return cli.Exit(fmt.Sprintf("not a directory: %s", absRoot), 2)
}
xm := meta.XattrMeta{}
err = xm.Test(absRoot)
if err != nil {
return err
}
var (
scanned int
converted int
skipped int
errCount int
)
walkErr := filepath.WalkDir(absRoot, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
errCount++
// keep going
return nil
}
rel, err := filepath.Rel(absRoot, path)
if err != nil {
errCount++
return nil
}
if rel == "." {
// skip root itself
return nil
}
attrs, err := xm.ListAttributes(absRoot, rel)
if err != nil {
errCount++
return nil
}
if len(attrs) == 0 {
// not an s3 object, do not track as skipped
return nil
}
scanned++
// Collect legacy metadata attributes.
oldAttrs := make([]string, 0)
for _, a := range attrs {
if strings.HasPrefix(a, oldMetadataHdr+".") {
oldAttrs = append(oldAttrs, a)
}
}
if len(oldAttrs) == 0 {
skipped++
return nil
}
// Build key/value map from legacy attrs.
md := make(map[string]string, len(oldAttrs))
for _, a := range oldAttrs {
b, err := xm.RetrieveAttribute(nil, absRoot, rel, a)
if err != nil {
// If we can't read one key, don't convert this entry.
errCount++
return nil
}
key := strings.TrimPrefix(a, oldMetadataHdr+".")
md[key] = string(b)
}
// Marshal to JSON and store as user.metadata.
j, err := json.Marshal(md)
if err != nil {
errCount++
return nil
}
if err := xm.StoreAttribute(nil, absRoot, rel, newMetadataAttr, j); err != nil {
errCount++
return nil
}
// Cleanup old metadata only after successful write of user.metadata.
for _, a := range oldAttrs {
if err := xm.DeleteAttribute(absRoot, rel, a); err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
// Count, but continue cleanup attempts.
errCount++
}
}
converted++
return nil
})
if walkErr != nil {
return fmt.Errorf("walk directory: %w", walkErr)
}
fmt.Printf(
"xattr metadata conversion is finished:\n directory: %s\n scanned: %d\n converted: %d\n skipped: %d\n errors: %d\n",
absRoot, scanned, converted, skipped, errCount,
)
return nil
}
tests/integration/CopyObject.go
+34 -33
View File
@@ -345,17 +345,17 @@ func CopyObject_should_replace_tagging(s *S3Conf) error {
func CopyObject_to_itself_with_new_metadata(s *S3Conf) error {
testName := "CopyObject_to_itself_with_new_metadata"
meta := map[string]string{
"Hello": "World",
}
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
_, err := putObjects(s3client, []string{obj}, bucket)
if err != nil {
return err
}
meta := map[string]string{
"Hello": "World",
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: &bucket,
@@ -710,8 +710,9 @@ func CopyObject_should_replace_meta_props(s *S3Conf) error {
cType, cEnc, cDesp, cLang := "application/binary", "hex", "desp", "mex"
cacheControl, expires := "no-cache", time.Now().Add(time.Hour*10)
meta := map[string]string{
"foo": "bar",
"baz": "quxx",
"foo": "bar",
"baz": "quxx",
strings.Repeat("d", 500): strings.Repeat("e", 400),
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
@@ -1180,19 +1181,19 @@ func CopyObject_with_metadata(s *S3Conf) error {
}
meta := map[string]string{
"Key": "Val",
"X-Test": "Example",
"UPPERCASE": "should-remain",
"MiXeD-CaSe": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"KeyURL": "https://example.com/test?query=1",
"EmptyValue": "",
"LongKeyNameThatShouldStillBeValidButQuiteLongToTestLimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"WhitespaceKey ": " trailing-key",
"Key": "Val",
"X-Test": "Example",
"UPPERCASE": "should-remain",
"MiXeD-CaSe": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"KeyURL": "https://example.com/test?query=1",
"EmptyValue": "",
strings.Repeat("v", 280): strings.Repeat("k", 300),
"WhitespaceKey ": " trailing-key",
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
@@ -1219,19 +1220,19 @@ func CopyObject_with_metadata(s *S3Conf) error {
}
expectedMeta := map[string]string{
"key": "Val",
"x-test": "Example",
"uppercase": "should-remain",
"mixed-case": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"keyurl": "https://example.com/test?query=1",
"emptyvalue": "",
"longkeynamethatshouldstillbevalidbutquitelongtotestlimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"whitespacekey": "trailing-key",
"key": "Val",
"x-test": "Example",
"uppercase": "should-remain",
"mixed-case": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"keyurl": "https://example.com/test?query=1",
"emptyvalue": "",
strings.Repeat("v", 280): strings.Repeat("k", 300),
"whitespacekey": "trailing-key",
}
if !areMapsSame(expectedMeta, res.Metadata) {
tests/integration/CreateMultipartUpload.go
+3 -2
View File
@@ -43,8 +43,9 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
meta := map[string]string{
"prop1": "val1",
"prop2": "val2",
"prop1": "val1",
"prop2": "val2",
strings.Repeat("b", 300): strings.Repeat("c", 500),
}
cType, cEnc, cDesp, cLang := "application/text", "testenc", "testdesp", "sp"
cacheControl, expires := "no-cache", time.Now().Add(time.Hour*5)
tests/integration/PutObject.go
+26 -26
View File
@@ -471,35 +471,35 @@ func PutObject_with_metadata(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
inputMeta := map[string]string{
"Key": "Val",
"X-Test": "Example",
"UPPERCASE": "should-remain",
"MiXeD-CaSe": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"KeyURL": "https://example.com/test?query=1",
"EmptyValue": "",
"LongKeyNameThatShouldStillBeValidButQuiteLongToTestLimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"WhitespaceKey ": " trailing-key",
"Key": "Val",
"X-Test": "Example",
"UPPERCASE": "should-remain",
"MiXeD-CaSe": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"KeyURL": "https://example.com/test?query=1",
"EmptyValue": "",
strings.Repeat("a", 256): "some long metadata value to ensure nothing breaks at higher header sizes",
"WhitespaceKey ": " trailing-key",
}
expectedMeta := map[string]string{
"key": "Val",
"x-test": "Example",
"uppercase": "should-remain",
"mixed-case": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"keyurl": "https://example.com/test?query=1",
"emptyvalue": "",
"longkeynamethatshouldstillbevalidbutquitelongtotestlimits": "some long metadata value to ensure nothing breaks at higher header sizes",
"whitespacekey": "trailing-key",
"key": "Val",
"x-test": "Example",
"uppercase": "should-remain",
"mixed-case": "normalize-to-lower",
"with-number-123": "numeric-test",
"123numeric-prefix": "value123",
"key_with_underscore": "underscore-ok",
"key-with-dash": "dash-ok",
"key.with.dot": "dot-ok",
"keyurl": "https://example.com/test?query=1",
"emptyvalue": "",
strings.Repeat("a", 256): "some long metadata value to ensure nothing breaks at higher header sizes",
"whitespacekey": "trailing-key",
}
for i, test := range []struct {