feat: Implements object meta properties for CopyObject in azure and posix backends.

Fixes #998 Closes #1125 Closes #1126 Closes #1127 Implements objects meta properties(Content-Disposition, Content-Language, Content-Encoding, Cache-Control, Expires) and tagging besed on the directives(metadata, tagging) in CopyObject in posix and azure backends. The properties/tagging should be coppied from the source object if "COPY" directive is provided and it should be replaced otherwise. Changes the object copy principle in azure: instead of using the `CopyFromURL` method from azure sdk, it first loads the object then creates one, to be able to compare and store the meta properties.
2026-09-19 14:34:19 +00:00 · 2025-03-17 09:37:05 -07:00
parent 38768a92b8
commit cfb2d6d87d
13 changed files with 819 additions and 61 deletions
@@ -457,7 +457,7 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G
 		ExpiresString:      blobDownloadResponse.Metadata[string(keyExpires)],
 		ETag:               (*string)(blobDownloadResponse.ETag),
 		LastModified:       blobDownloadResponse.LastModified,
-		Metadata:           parseAzMetadata(blobDownloadResponse.Metadata),
+		Metadata:           parseAndFilterAzMetadata(blobDownloadResponse.Metadata),
 		TagCount:           &tagcount,
 		ContentRange:       blobDownloadResponse.ContentRange,
 		Body:               blobDownloadResponse.Body,
@@ -519,7 +519,7 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3
 		ExpiresString:      resp.Metadata[string(keyExpires)],
 		ETag:               (*string)(resp.ETag),
 		LastModified:       resp.LastModified,
-		Metadata:           parseAzMetadata(resp.Metadata),
+		Metadata:           parseAndFilterAzMetadata(resp.Metadata),
 		StorageClass:       types.StorageClassStandard,
 	}

@@ -767,41 +767,158 @@ func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput
 	}, nil
 }

-func (az *Azure) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	bclient, err := az.getBlobClient(*input.Bucket, *input.Key)
+func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	dstClient, err := az.getBlobClient(*input.Bucket, *input.Key)
 	if err != nil {
 		return nil, err
 	}
-
 	if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource {
-		props, err := bclient.GetProperties(ctx, nil)
+		if input.MetadataDirective != types.MetadataDirectiveReplace {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+		}
+
+		// Set object meta http headers
+		res, err := dstClient.SetHTTPHeaders(ctx, blob.HTTPHeaders{
+			BlobCacheControl:       input.CacheControl,
+			BlobContentDisposition: input.ContentDisposition,
+			BlobContentEncoding:    input.ContentEncoding,
+			BlobContentLanguage:    input.ContentLanguage,
+			BlobContentType:        input.ContentType,
+		}, nil)
 		if err != nil {
 			return nil, azureErrToS3Err(err)
 		}

-		mdmap := props.Metadata
-		if isMetaSame(mdmap, input.Metadata) {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
+		meta := input.Metadata
+		if meta == nil {
+			meta = make(map[string]string)
 		}
+
+		// Embed "Expires" in object metadata
+		if getString(input.Expires) != "" {
+			meta[string(keyExpires)] = *input.Expires
+		}
+		// Set object metadata
+		_, err = dstClient.SetMetadata(ctx, parseMetadata(meta), nil)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+
+		// Set object legal hold
+		if input.ObjectLockLegalHoldStatus != "" {
+			err = az.PutObjectLegalHold(ctx, *input.Bucket, *input.Key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn)
+			if err != nil {
+				return nil, azureErrToS3Err(err)
+			}
+		}
+		// Set object retention
+		if input.ObjectLockMode != "" && input.ObjectLockRetainUntilDate != nil {
+			retention := s3response.PutObjectRetentionInput{
+				Mode: types.ObjectLockRetentionMode(input.ObjectLockMode),
+				RetainUntilDate: s3response.AmzDate{
+					Time: *input.ObjectLockRetainUntilDate,
+				},
+			}
+
+			retParsed, err := json.Marshal(retention)
+			if err != nil {
+				return nil, fmt.Errorf("parse object retention: %w", err)
+			}
+			err = az.PutObjectRetention(ctx, *input.Bucket, *input.Key, "", true, retParsed)
+			if err != nil {
+				return nil, azureErrToS3Err(err)
+			}
+		}
+
+		// Set object Tagging, if tagging directive is "REPLACE"
+		if input.TaggingDirective == types.TaggingDirectiveReplace {
+			tags, err := parseTags(input.Tagging)
+			if err != nil {
+				return nil, err
+			}
+			_, err = dstClient.SetTags(ctx, tags, nil)
+			if err != nil {
+				return nil, azureErrToS3Err(err)
+			}
+		}
+
+		return &s3.CopyObjectOutput{
+			CopyObjectResult: &types.CopyObjectResult{
+				LastModified: res.LastModified,
+				ETag:         (*string)(res.ETag),
+			},
+		}, nil
 	}

-	tags, err := parseTags(input.Tagging)
+	srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
 	if err != nil {
 		return nil, err
 	}

-	resp, err := bclient.CopyFromURL(ctx, az.serviceURL+"/"+*input.CopySource, &blob.CopyFromURLOptions{
-		BlobTags: tags,
-		Metadata: parseMetadata(input.Metadata),
-	})
+	// Get the source object
+	downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
 	if err != nil {
 		return nil, azureErrToS3Err(err)
 	}

+	pInput := s3response.PutObjectInput{
+		Body:                      downloadResp.Body,
+		Bucket:                    input.Bucket,
+		Key:                       input.Key,
+		ContentLength:             downloadResp.ContentLength,
+		ContentType:               input.ContentType,
+		ContentEncoding:           input.ContentEncoding,
+		ContentDisposition:        input.ContentDisposition,
+		ContentLanguage:           input.ContentLanguage,
+		CacheControl:              input.CacheControl,
+		Expires:                   input.Expires,
+		Metadata:                  input.Metadata,
+		ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate,
+		ObjectLockMode:            input.ObjectLockMode,
+		ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus,
+	}
+
+	if input.MetadataDirective == types.MetadataDirectiveCopy {
+		// Expires is in downloadResp.Metadata
+		pInput.Expires = nil
+		pInput.CacheControl = downloadResp.CacheControl
+		pInput.ContentDisposition = downloadResp.ContentDisposition
+		pInput.ContentEncoding = downloadResp.ContentEncoding
+		pInput.ContentLanguage = downloadResp.ContentLanguage
+		pInput.ContentType = downloadResp.ContentType
+		pInput.Metadata = parseAzMetadata(downloadResp.Metadata)
+	}
+
+	if input.TaggingDirective == types.TaggingDirectiveReplace {
+		pInput.Tagging = input.Tagging
+	}
+
+	// Create the destination object
+	resp, err := az.PutObject(ctx, pInput)
+	if err != nil {
+		return nil, err
+	}
+
+	// Copy the object tagging, if tagging directive is "COPY"
+	if input.TaggingDirective == types.TaggingDirectiveCopy {
+		srcClient, err := az.getBlobClient(srcBucket, srcObj)
+		if err != nil {
+			return nil, err
+		}
+		res, err := srcClient.GetTags(ctx, nil)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+
+		_, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil)
+		if err != nil {
+			return nil, azureErrToS3Err(err)
+		}
+	}
+
 	return &s3.CopyObjectOutput{
 		CopyObjectResult: &types.CopyObjectResult{
-			ETag:         (*string)(resp.ETag),
-			LastModified: resp.LastModified,
+			ETag: &resp.ETag,
 		},
 	}, nil
 }
@@ -1629,7 +1746,7 @@ func parseMetadata(m map[string]string) map[string]*string {
 	return meta
 }

-func parseAzMetadata(m map[string]*string) map[string]string {
+func parseAndFilterAzMetadata(m map[string]*string) map[string]string {
 	if m == nil {
 		return nil
 	}
@@ -1648,6 +1765,19 @@ func parseAzMetadata(m map[string]*string) map[string]string {
 	return meta
 }

+func parseAzMetadata(m map[string]*string) map[string]string {
+	if m == nil {
+		return nil
+	}
+
+	meta := make(map[string]string)
+
+	for k, v := range m {
+		meta[k] = *v
+	}
+	return meta
+}
+
 func parseTags(tagstr *string) (map[string]string, error) {
 	tagsStr := getString(tagstr)
 	tags := make(map[string]string)
@@ -1830,24 +1960,6 @@ func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
 	return &acl, nil
 }

-func isMetaSame(azMeta map[string]*string, awsMeta map[string]string) bool {
-	if len(azMeta) != len(awsMeta) {
-		return false
-	}
-
-	for key, val := range azMeta {
-		if key == string(keyAclCapital) || key == string(keyAclLower) {
-			continue
-		}
-		awsVal, ok := awsMeta[key]
-		if !ok || awsVal != *val {
-			return false
-		}
-	}
-
-	return true
-}
-
 func createMetaTmpPath(obj, uploadId string) string {
 	objNameSum := sha256.Sum256([]byte(obj))
 	return filepath.Join(string(metaTmpMultipartPrefix), uploadId, fmt.Sprintf("%x", objNameSum))
@@ -62,7 +62,7 @@ type Backend interface {
 	GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
 	GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
 	GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
-	CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
+	CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
 	ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
 	ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
 	DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
@@ -188,7 +188,7 @@ func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (
 func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
 	return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
@@ -211,6 +211,30 @@ func ParseCopySource(copySourceHeader string) (string, string, string, error) {
 	return srcBucket, srcObject, versionId, nil
 }

+// ParseObjectTags parses the url encoded input string into
+// map[string]string key-value tag set
+func ParseObjectTags(t string) (map[string]string, error) {
+	tagging := make(map[string]string)
+
+	if t == "" {
+		return tagging, nil
+	}
+
+	tagParts := strings.Split(t, "&")
+	for _, prt := range tagParts {
+		p := strings.Split(prt, "=")
+		if len(p) != 2 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+		}
+		if len(p[0]) > 128 || len(p[1]) > 256 {
+			return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
+		}
+		tagging[p[0]] = p[1]
+	}
+
+	return tagging, nil
+}
+
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {
@@ -3819,7 +3819,7 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr
 	}, nil
 }

-func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	if input.Bucket == nil {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
 	}
@@ -3925,6 +3925,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 			return &s3.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
 		}

+		// Delete the object metadata
 		for k := range mdmap {
 			err := p.meta.DeleteAttribute(dstBucket, dstObject,
 				fmt.Sprintf("%v.%v", metaHdr, k))
@@ -3932,6 +3933,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 				return nil, fmt.Errorf("delete user metadata: %w", err)
 			}
 		}
+		// Store the new metadata
 		for k, v := range input.Metadata {
 			err := p.meta.StoreAttribute(nil, dstBucket, dstObject,
 				fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
@@ -4006,6 +4008,32 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 			return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
 		}
 		version = backend.GetPtrFromString(string(vId))
+
+		// Store the provided object meta properties
+		err = p.storeObjectMetadata(nil, dstBucket, dstObject,
+			objectMetadata{
+				ContentType:        input.ContentType,
+				ContentEncoding:    input.ContentEncoding,
+				ContentLanguage:    input.ContentLanguage,
+				ContentDisposition: input.ContentDisposition,
+				CacheControl:       input.CacheControl,
+				Expires:            input.Expires,
+			})
+		if err != nil {
+			return nil, err
+		}
+
+		if input.TaggingDirective == types.TaggingDirectiveReplace {
+			tags, err := backend.ParseObjectTags(getString(input.Tagging))
+			if err != nil {
+				return nil, err
+			}
+
+			err = p.PutObjectTagging(ctx, dstBucket, dstObject, tags)
+			if err != nil {
+				return nil, err
+			}
+		}
 	} else {
 		contentLength := fi.Size()

@@ -4020,18 +4048,61 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 			checksums.Algorithm = input.ChecksumAlgorithm
 		}

-		res, err := p.PutObject(ctx,
-			s3response.PutObjectInput{
-				Bucket:            &dstBucket,
-				Key:               &dstObject,
-				Body:              f,
-				ContentLength:     &contentLength,
-				Metadata:          input.Metadata,
-				ChecksumAlgorithm: checksums.Algorithm,
-			})
+		putObjectInput := s3response.PutObjectInput{
+			Bucket:                    &dstBucket,
+			Key:                       &dstObject,
+			Body:                      f,
+			ContentLength:             &contentLength,
+			ChecksumAlgorithm:         checksums.Algorithm,
+			ContentType:               input.ContentType,
+			ContentEncoding:           input.ContentEncoding,
+			ContentDisposition:        input.ContentDisposition,
+			ContentLanguage:           input.ContentLanguage,
+			CacheControl:              input.CacheControl,
+			Expires:                   input.Expires,
+			Metadata:                  input.Metadata,
+			ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate,
+			ObjectLockMode:            input.ObjectLockMode,
+			ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus,
+		}
+
+		// load and pass the source object meta properties, if metadata directive is "COPY"
+		if input.MetadataDirective != types.MetadataDirectiveReplace {
+			metaProps := p.loadObjectMetaData(srcBucket, srcObject, &fi, nil)
+			putObjectInput.ContentEncoding = metaProps.ContentEncoding
+			putObjectInput.ContentDisposition = metaProps.ContentDisposition
+			putObjectInput.ContentLanguage = metaProps.ContentLanguage
+			putObjectInput.ContentType = metaProps.ContentType
+			putObjectInput.CacheControl = metaProps.CacheControl
+			putObjectInput.Expires = metaProps.Expires
+			putObjectInput.Metadata = mdmap
+		}
+
+		// pass the input tagging to PutObject, if tagging directive is "REPLACE"
+		if input.TaggingDirective == types.TaggingDirectiveReplace {
+			putObjectInput.Tagging = input.Tagging
+		}
+
+		res, err := p.PutObject(ctx, putObjectInput)
 		if err != nil {
 			return nil, err
 		}
+
+		// copy the source object tagging after the destination object
+		// creation, if tagging directive is "COPY"
+		if input.TaggingDirective == types.TaggingDirectiveCopy {
+			tagging, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, tagHdr)
+			if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+				return nil, fmt.Errorf("get source object tagging: %w", err)
+			}
+			if err == nil {
+				err := p.meta.StoreAttribute(nil, dstBucket, dstObject, tagHdr, tagging)
+				if err != nil {
+					return nil, fmt.Errorf("set destination object tagging: %w", err)
+				}
+			}
+		}
+
 		etag = res.ETag
 		version = &res.VersionID
 		crc32 = res.ChecksumCRC32
@@ -988,7 +988,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
 	}, handleError(err)
 }

-func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+func (s *S3Proxy) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
 	if input.CacheControl != nil && *input.CacheControl == "" {
 		input.CacheControl = nil
 	}
@@ -1031,7 +1031,7 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s
 	if input.ExpectedSourceBucketOwner != nil && *input.ExpectedSourceBucketOwner == "" {
 		input.ExpectedSourceBucketOwner = nil
 	}
-	if input.Expires != nil && *input.Expires == defTime {
+	if input.Expires != nil && *input.Expires == "" {
 		input.Expires = nil
 	}
 	if input.GrantFullControl != nil && *input.GrantFullControl == "" {
@@ -1071,7 +1071,58 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s
 		input.WebsiteRedirectLocation = nil
 	}

-	out, err := s.client.CopyObject(ctx, input)
+	var expires *time.Time
+	if input.Expires != nil {
+		exp, err := time.Parse(time.RFC1123, *input.Expires)
+		if err == nil {
+			expires = &exp
+		}
+	}
+
+	out, err := s.client.CopyObject(ctx,
+		&s3.CopyObjectInput{
+			Metadata:                       input.Metadata,
+			Bucket:                         input.Bucket,
+			CopySource:                     input.CopySource,
+			Key:                            input.Key,
+			CacheControl:                   input.CacheControl,
+			ContentDisposition:             input.ContentDisposition,
+			ContentEncoding:                input.ContentEncoding,
+			ContentLanguage:                input.ContentLanguage,
+			ContentType:                    input.ContentType,
+			CopySourceIfMatch:              input.CopySourceIfMatch,
+			CopySourceIfNoneMatch:          input.CopySourceIfNoneMatch,
+			CopySourceSSECustomerAlgorithm: input.CopySourceSSECustomerAlgorithm,
+			CopySourceSSECustomerKey:       input.CopySourceSSECustomerKey,
+			CopySourceSSECustomerKeyMD5:    input.CopySourceSSECustomerKeyMD5,
+			ExpectedBucketOwner:            input.ExpectedBucketOwner,
+			ExpectedSourceBucketOwner:      input.ExpectedSourceBucketOwner,
+			Expires:                        expires,
+			GrantFullControl:               input.GrantFullControl,
+			GrantRead:                      input.GrantRead,
+			GrantReadACP:                   input.GrantReadACP,
+			GrantWriteACP:                  input.GrantWriteACP,
+			SSECustomerAlgorithm:           input.SSECustomerAlgorithm,
+			SSECustomerKey:                 input.SSECustomerKey,
+			SSECustomerKeyMD5:              input.SSECustomerKeyMD5,
+			SSEKMSEncryptionContext:        input.SSEKMSEncryptionContext,
+			SSEKMSKeyId:                    input.SSEKMSKeyId,
+			Tagging:                        input.Tagging,
+			WebsiteRedirectLocation:        input.WebsiteRedirectLocation,
+			CopySourceIfModifiedSince:      input.CopySourceIfModifiedSince,
+			CopySourceIfUnmodifiedSince:    input.CopySourceIfUnmodifiedSince,
+			ObjectLockRetainUntilDate:      input.ObjectLockRetainUntilDate,
+			BucketKeyEnabled:               input.BucketKeyEnabled,
+			ACL:                            input.ACL,
+			ChecksumAlgorithm:              input.ChecksumAlgorithm,
+			MetadataDirective:              input.MetadataDirective,
+			ObjectLockLegalHoldStatus:      input.ObjectLockLegalHoldStatus,
+			ObjectLockMode:                 input.ObjectLockMode,
+			RequestPayer:                   input.RequestPayer,
+			ServerSideEncryption:           input.ServerSideEncryption,
+			StorageClass:                   input.StorageClass,
+			TaggingDirective:               input.TaggingDirective,
+		})
 	return out, handleError(err)
 }