Merge pull request #2027 from anaelorlinski/fix-multipart-upload

fix for multipart upload when using sidecar meta

backend/meta/meta.go

@@ -39,4 +39,9 @@ type MetadataStorer interface {
 	// DeleteAttributes removes all attributes for an object or a bucket.
 	// Returns an error if the operation fails.
 	DeleteAttributes(bucket, object string) error
+
+	// RenameObject renames all stored metadata from oldObject to newObject
+	// within the given bucket. This must be called whenever the data
+	// directory for an object is renamed so that metadata stays in sync.
+	RenameObject(bucket, oldObject, newObject string) error
 }

backend/meta/none.go

@@ -52,3 +52,8 @@ func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
 func (NoMeta) DeleteAttributes(bucket, object string) error {
 	return nil
 }
+
+// RenameObject is a no-op because NoMeta does not store metadata.
+func (NoMeta) RenameObject(_, _, _ string) error {
+	return nil
+}

backend/meta/sidecar.go

@@ -153,6 +153,25 @@ func (s SideCar) DeleteAttributes(bucket, object string) error {
 	return nil
 }
 
+// RenameObject renames the sidecar metadata directory from oldObject to
+// newObject so that path-based lookups continue to work after the data
+// directory has been renamed.
+func (s SideCar) RenameObject(bucket, oldObject, newObject string) error {
+	oldPath := filepath.Join(s.dir, bucket, oldObject)
+	newPath := filepath.Join(s.dir, bucket, newObject)
+
+	if err := os.MkdirAll(filepath.Dir(newPath), 0777); err != nil {
+		return fmt.Errorf("create parent for renamed metadata: %w", err)
+	}
+
+	err := os.Rename(oldPath, newPath)
+	if errors.Is(err, os.ErrNotExist) {
+		// No metadata stored yet — nothing to rename.
+		return nil
+	}
+	return err
+}
+
 func (s SideCar) cleanupEmptyDirs(metadir, bucket, object string) {
 	removeIfEmpty(metadir)
 	if bucket == "" {

backend/meta/xattr.go

@@ -85,6 +85,12 @@ func (x XattrMeta) DeleteAttributes(bucket, object string) error {
 	return nil
 }
 
+// RenameObject is a no-op for xattr because extended attributes are stored
+// on the inodes and follow the file/directory when it is renamed.
+func (x XattrMeta) RenameObject(_, _, _ string) error {
+	return nil
+}
+
 // ListAttributes lists all attributes for an object in a bucket.
 func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
 	attrs, err := xattr.List(filepath.Join(bucket, object))

backend/posix/posix.go

@@ -1698,9 +1698,22 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
 		}
 		return res, "", fmt.Errorf("mark upload in-progress: %w", err)
 	}
+
+	// Rename sidecar metadata to match the new data directory path.
+	// For xattr this is a no-op since attributes follow the inode.
+	metaObjDir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
+	oldMetaObj := filepath.Join(metaObjDir, uploadID)
+	newMetaObj := filepath.Join(metaObjDir, activeUploadName)
+	if err := p.meta.RenameObject(bucket, oldMetaObj, newMetaObj); err != nil {
+		// Roll back the data directory rename so a future retry can succeed.
+		os.Rename(uploadIDInProgress, uploadIDDir)
+		return res, "", fmt.Errorf("rename metadata for in-progress: %w", err)
+	}
+
 	// Best-effort rename back on failure so a future retry can still complete.
 	// On success, os.RemoveAll below removes uploadIDInProgress so this is a no-op.
 	defer os.Rename(uploadIDInProgress, uploadIDDir)
+	defer p.meta.RenameObject(bucket, newMetaObj, oldMetaObj)
 
 	b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	if err == nil || errors.Is(err, fs.ErrNotExist) || errors.Is(err, meta.ErrNoSuchKey) {