diff --git a/seaweed-volume/proto/volume_server.proto b/seaweed-volume/proto/volume_server.proto index d9aeb2604..c63177cb4 100644 --- a/seaweed-volume/proto/volume_server.proto +++ b/seaweed-volume/proto/volume_server.proto @@ -458,6 +458,7 @@ message VolumeEcShardsMountRequest { uint32 volume_id = 1; string collection = 2; repeated uint32 shard_ids = 3; + string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423) } message VolumeEcShardsMountResponse { } diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index 4e7ce0556..10632443e 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -2612,7 +2612,7 @@ impl VolumeServer for VolumeGrpcService { let mut store = self.state.store.write().unwrap(); for &shard_id in &req.shard_ids { store - .mount_ec_shard(vid, &req.collection, shard_id) + .mount_ec_shard(vid, &req.collection, shard_id, &req.source_disk_type) .map_err(|e| { Status::internal(format!("mount {}.{}: {}", req.volume_id, shard_id, e)) })?; diff --git a/seaweed-volume/src/server/heartbeat.rs b/seaweed-volume/src/server/heartbeat.rs index b551ac4e7..ceed62696 100644 --- a/seaweed-volume/src/server/heartbeat.rs +++ b/seaweed-volume/src/server/heartbeat.rs @@ -1241,7 +1241,7 @@ mod tests { let shard_path = format!("{}/ec_metrics_case_27.ec00", dir); std::fs::write(&shard_path, b"ec-shard").unwrap(); store.locations[0] - .mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0]) + .mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0], "") .unwrap(); let state = test_state_with_store(store); @@ -1283,7 +1283,7 @@ mod tests { std::fs::write(format!("{}/expired_heartbeat_ec_31.ec00", dir), b"expired").unwrap(); store.locations[0] - .mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0]) + .mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0], "") .unwrap(); store .find_ec_volume_mut(VolumeId(31)) @@ -1586,7 +1586,7 @@ mod tests { std::fs::write(format!("{}/ec_delta_case_81.ec00", dir), b"delta").unwrap(); store.locations[0] - .mount_ec_shards(VolumeId(81), "ec_delta_case", &[0]) + .mount_ec_shards(VolumeId(81), "ec_delta_case", &[0], "") .unwrap(); let current = collect_ec_shard_delta_messages(&store); let (new_ec_shards, deleted_ec_shards) = diff --git a/seaweed-volume/src/storage/disk_location.rs b/seaweed-volume/src/storage/disk_location.rs index ddd2584f1..799f3c1ea 100644 --- a/seaweed-volume/src/storage/disk_location.rs +++ b/seaweed-volume/src/storage/disk_location.rs @@ -600,14 +600,22 @@ impl DiskLocation { } /// Mount EC shards for a volume on this location. + /// + /// `source_disk_type` is the source volume's disk type carried on the + /// `VolumeEcShardsMount` RPC. When non-empty it overrides the in-memory + /// EC volume's reported disk type so heartbeats keep reporting under + /// the source's disk type after EC encoding (#9423). Empty means "use + /// this location's disk type" — used by disk-scan reload paths that + /// have no orchestrator context. pub fn mount_ec_shards( &mut self, vid: VolumeId, collection: &str, shard_ids: &[u32], + source_disk_type: &str, ) -> Result<(), VolumeError> { let idx_dir = self.idx_directory.clone(); - self.mount_ec_shards_with_idx_dir(vid, collection, shard_ids, &idx_dir) + self.mount_ec_shards_with_idx_dir(vid, collection, shard_ids, &idx_dir, source_disk_type) } /// Mount EC shards but explicitly specify the idx directory the @@ -627,6 +635,7 @@ impl DiskLocation { collection: &str, shard_ids: &[u32], idx_dir: &str, + source_disk_type: &str, ) -> Result<(), VolumeError> { let dir = self.directory.clone(); // Avoid the entry().or_insert_with() pattern here: that closure @@ -643,10 +652,22 @@ impl DiskLocation { .ec_volumes .get_mut(&vid) .expect("just inserted above"); - ec_vol.disk_type = self.disk_type.clone(); + // When the orchestrator supplied a source disk type on the Mount + // RPC, override the EC volume's disk type so heartbeats report + // under the source volume's disk type (#9423). When the caller + // passed "" (disk-scan reload, orphan-shard reconcile, restart) + // we only default to this location's disk type for a fresh EC + // volume; otherwise we leave whatever a prior RPC mount set in + // place, so empty-source reloads don't reintroduce the drift. + if !source_disk_type.is_empty() { + ec_vol.set_disk_type(DiskType::from_string(source_disk_type)); + } else if ec_vol.shard_count() == 0 { + ec_vol.set_disk_type(self.disk_type.clone()); + } for &shard_id in shard_ids { - let shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8); + let mut shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8); + shard.disk_type = ec_vol.disk_type.clone(); ec_vol.add_shard(shard).map_err(VolumeError::Io)?; crate::metrics::VOLUME_GAUGE .with_label_values(&[collection, "ec_shards"]) @@ -804,7 +825,7 @@ impl DiskLocation { } let shard_ids: Vec = shards.iter().map(|(_, sid)| *sid).collect(); - if let Err(e) = self.mount_ec_shards(vid, collection, &shard_ids) { + if let Err(e) = self.mount_ec_shards(vid, collection, &shard_ids, "") { // mount_ec_shards adds shards one at a time and increments // the per-shard metric for each. If it fails halfway, plain // ec_volumes.remove(vid) would leak metric increments for @@ -1237,7 +1258,7 @@ mod tests { let shard_path = format!("{}/pics_7.ec00", dir); std::fs::write(&shard_path, b"ec-shard").unwrap(); - loc.mount_ec_shards(VolumeId(7), "pics", &[0]).unwrap(); + loc.mount_ec_shards(VolumeId(7), "pics", &[0], "").unwrap(); assert!(loc.has_ec_volume(VolumeId(7))); assert!(std::path::Path::new(&shard_path).exists()); assert!(std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists()); @@ -1249,6 +1270,56 @@ mod tests { assert!(!std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists()); } + /// #9423: after an RPC `VolumeEcShardsMount` records the source + /// volume's disk type on the EcVolume, a later empty-source mount + /// call (disk-scan reload, orphan-shard reconcile, restart) must + /// not clobber that override back to the physical location's disk + /// type — otherwise heartbeat reporting drifts back to "hdd" on + /// every reload. + #[test] + fn test_mount_ec_shards_empty_source_preserves_existing_disk_type() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let mut loc = DiskLocation::new( + dir, + dir, + 10, + DiskType::HardDrive, + MinFreeSpace::Percent(1.0), + Vec::new(), + ) + .unwrap(); + + // Plant the first shard so the EC volume can mount, then call + // mount_ec_shards with source_disk_type="ssd" — simulating the + // VolumeEcShardsMount RPC path. + std::fs::write(format!("{}/pics_7.ec00", dir), b"ec-shard").unwrap(); + loc.mount_ec_shards(VolumeId(7), "pics", &[0], "ssd").unwrap(); + { + let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume mounted"); + assert_eq!( + ec_vol.disk_type, + DiskType::Ssd, + "first RPC mount should have set disk type to source's value", + ); + } + + // Plant another shard and re-mount with an empty source — + // matching what a disk-scan reload or orphan-shard reconcile + // would pass. The previously-recorded "ssd" override must + // survive. + std::fs::write(format!("{}/pics_7.ec01", dir), b"ec-shard").unwrap(); + loc.mount_ec_shards(VolumeId(7), "pics", &[1], "").unwrap(); + { + let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume still mounted"); + assert_eq!( + ec_vol.disk_type, + DiskType::Ssd, + "empty-source remount must not reset disk type to the physical location's", + ); + } + } + #[test] fn test_disk_location_persists_directory_uuid_and_tags() { let tmp = TempDir::new().unwrap(); diff --git a/seaweed-volume/src/storage/erasure_coding/ec_volume.rs b/seaweed-volume/src/storage/erasure_coding/ec_volume.rs index 38c411c29..3e0b68f7f 100644 --- a/seaweed-volume/src/storage/erasure_coding/ec_volume.rs +++ b/seaweed-volume/src/storage/erasure_coding/ec_volume.rs @@ -327,6 +327,20 @@ impl EcVolume { Ok(()) } + /// Override the disk type the EC volume (and its already-mounted + /// shards) reports under. Used by the `VolumeEcShardsMount` handler + /// so the source volume's disk type is preserved across encoding + /// (#9423). Not persisted across restarts — disk-scan reload paths + /// default to the physical location's disk type. + pub fn set_disk_type(&mut self, d: DiskType) { + self.disk_type = d.clone(); + for slot in self.shards.iter_mut() { + if let Some(shard) = slot { + shard.disk_type = d.clone(); + } + } + } + /// Remove and close a shard. pub fn remove_shard(&mut self, shard_id: ShardId) { if let Some(ref mut shard) = self.shards[shard_id as usize] { diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index 02d42bab5..cc1f93e3d 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -636,22 +636,25 @@ impl Store { )) })?; - self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids) + self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids, "") } /// Mount a single EC shard, searching all locations for the shard file. /// Matches Go's Store.MountEcShards which mounts one shard at a time. + /// `source_disk_type` is the source volume's disk type from the + /// `VolumeEcShardsMount` RPC (#9423); pass `""` for non-RPC paths. pub fn mount_ec_shard( &mut self, vid: VolumeId, collection: &str, shard_id: u32, + source_disk_type: &str, ) -> Result<(), VolumeError> { for loc in &mut self.locations { // Check if the shard file exists on this location let shard = EcVolumeShard::new(&loc.directory, collection, vid, shard_id as u8); if std::path::Path::new(&shard.file_name()).exists() { - loc.mount_ec_shards(vid, collection, &[shard_id])?; + loc.mount_ec_shards(vid, collection, &[shard_id], source_disk_type)?; return Ok(()); } } @@ -1481,7 +1484,7 @@ mod tests { std::fs::write(format!("{}/expired_ec_case_9.ec00", dir), b"expired").unwrap(); store.locations[0] - .mount_ec_shards(VolumeId(9), "expired_ec_case", &[0]) + .mount_ec_shards(VolumeId(9), "expired_ec_case", &[0], "") .unwrap(); store.find_ec_volume_mut(VolumeId(9)).unwrap().expire_at_sec = 1; @@ -1576,7 +1579,7 @@ mod tests { ) .unwrap(); store.locations[1] - .mount_ec_shards(vid, collection, &[0]) + .mount_ec_shards(vid, collection, &[0], "") .unwrap(); // Stray .ecx on disk 2 must not win. @@ -1643,7 +1646,7 @@ mod tests { ) .unwrap(); store.locations[1] - .mount_ec_shards(vid, collection, &[0]) + .mount_ec_shards(vid, collection, &[0], "") .unwrap(); let got = store.find_ec_shard_target_location(collection, vid, 10); diff --git a/seaweed-volume/src/storage/store_ec_reconcile.rs b/seaweed-volume/src/storage/store_ec_reconcile.rs index 77771c0f8..8c75bd63d 100644 --- a/seaweed-volume/src/storage/store_ec_reconcile.rs +++ b/seaweed-volume/src/storage/store_ec_reconcile.rs @@ -115,6 +115,7 @@ impl Store { &key.collection, &shard_ids, &owner_idx_dir, + "", ) { // mount_ec_shards_with_idx_dir adds shards one at a // time and increments the `ec_shards` gauge per shard diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index d9aeb2604..c63177cb4 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -458,6 +458,7 @@ message VolumeEcShardsMountRequest { uint32 volume_id = 1; string collection = 2; repeated uint32 shard_ids = 3; + string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423) } message VolumeEcShardsMountResponse { } diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 6fe1d6686..4d2d37df5 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -3632,12 +3632,13 @@ func (*VolumeEcShardsDeleteResponse) Descriptor() ([]byte, []int) { } type VolumeEcShardsMountRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` - Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"` - ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"` + ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"` + SourceDiskType string `protobuf:"bytes,4,opt,name=source_disk_type,json=sourceDiskType,proto3" json:"source_disk_type,omitempty"` // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *VolumeEcShardsMountRequest) Reset() { @@ -3691,6 +3692,13 @@ func (x *VolumeEcShardsMountRequest) GetShardIds() []uint32 { return nil } +func (x *VolumeEcShardsMountRequest) GetSourceDiskType() string { + if x != nil { + return x.SourceDiskType + } + return "" +} + type VolumeEcShardsMountResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -7004,13 +7012,14 @@ const file_volume_server_proto_rawDesc = "" + "collection\x18\x02 \x01(\tR\n" + "collection\x12\x1b\n" + "\tshard_ids\x18\x03 \x03(\rR\bshardIds\"\x1e\n" + - "\x1cVolumeEcShardsDeleteResponse\"v\n" + + "\x1cVolumeEcShardsDeleteResponse\"\xa0\x01\n" + "\x1aVolumeEcShardsMountRequest\x12\x1b\n" + "\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1e\n" + "\n" + "collection\x18\x02 \x01(\tR\n" + "collection\x12\x1b\n" + - "\tshard_ids\x18\x03 \x03(\rR\bshardIds\"\x1d\n" + + "\tshard_ids\x18\x03 \x03(\rR\bshardIds\x12(\n" + + "\x10source_disk_type\x18\x04 \x01(\tR\x0esourceDiskType\"\x1d\n" + "\x1bVolumeEcShardsMountResponse\"X\n" + "\x1cVolumeEcShardsUnmountRequest\x12\x1b\n" + "\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1b\n" + diff --git a/weed/pb/worker.proto b/weed/pb/worker.proto index 535b9746c..2d2c6b2d1 100644 --- a/weed/pb/worker.proto +++ b/weed/pb/worker.proto @@ -170,12 +170,14 @@ message VacuumTaskParams { // ErasureCodingTaskParams for EC encoding operations message ErasureCodingTaskParams { + reserved 7; uint64 estimated_shard_size = 1; // Estimated size per shard int32 data_shards = 2; // Number of data shards (default: 10) int32 parity_shards = 3; // Number of parity shards (default: 4) string working_dir = 4; // Working directory for EC processing string master_client = 5; // Master server address bool cleanup_source = 6; // Whether to cleanup source volume after EC + string source_disk_type = 8; // Source volume's disk type, passed to VolumeEcShardsMount so shards report under it (#9423) } // TaskSource represents a unified source location for any task type diff --git a/weed/pb/worker_pb/worker.pb.go b/weed/pb/worker_pb/worker.pb.go index c87c8fff4..a978455da 100644 --- a/weed/pb/worker_pb/worker.pb.go +++ b/weed/pb/worker_pb/worker.pb.go @@ -1322,6 +1322,7 @@ type ErasureCodingTaskParams struct { WorkingDir string `protobuf:"bytes,4,opt,name=working_dir,json=workingDir,proto3" json:"working_dir,omitempty"` // Working directory for EC processing MasterClient string `protobuf:"bytes,5,opt,name=master_client,json=masterClient,proto3" json:"master_client,omitempty"` // Master server address CleanupSource bool `protobuf:"varint,6,opt,name=cleanup_source,json=cleanupSource,proto3" json:"cleanup_source,omitempty"` // Whether to cleanup source volume after EC + SourceDiskType string `protobuf:"bytes,8,opt,name=source_disk_type,json=sourceDiskType,proto3" json:"source_disk_type,omitempty"` // Source volume's disk type, passed to VolumeEcShardsMount so shards report under it (#9423) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1398,6 +1399,13 @@ func (x *ErasureCodingTaskParams) GetCleanupSource() bool { return false } +func (x *ErasureCodingTaskParams) GetSourceDiskType() string { + if x != nil { + return x.SourceDiskType + } + return "" +} + // TaskSource represents a unified source location for any task type type TaskSource struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -4032,7 +4040,7 @@ const file_worker_proto_rawDesc = "" + "batch_size\x18\x03 \x01(\x05R\tbatchSize\x12\x1f\n" + "\vworking_dir\x18\x04 \x01(\tR\n" + "workingDir\x12'\n" + - "\x0fverify_checksum\x18\x05 \x01(\bR\x0everifyChecksum\"\xfe\x01\n" + + "\x0fverify_checksum\x18\x05 \x01(\bR\x0everifyChecksum\"\xae\x02\n" + "\x17ErasureCodingTaskParams\x120\n" + "\x14estimated_shard_size\x18\x01 \x01(\x04R\x12estimatedShardSize\x12\x1f\n" + "\vdata_shards\x18\x02 \x01(\x05R\n" + @@ -4041,7 +4049,8 @@ const file_worker_proto_rawDesc = "" + "\vworking_dir\x18\x04 \x01(\tR\n" + "workingDir\x12#\n" + "\rmaster_client\x18\x05 \x01(\tR\fmasterClient\x12%\n" + - "\x0ecleanup_source\x18\x06 \x01(\bR\rcleanupSource\"\xcf\x01\n" + + "\x0ecleanup_source\x18\x06 \x01(\bR\rcleanupSource\x12(\n" + + "\x10source_disk_type\x18\b \x01(\tR\x0esourceDiskTypeJ\x04\b\a\x10\b\"\xcf\x01\n" + "\n" + "TaskSource\x12\x12\n" + "\x04node\x18\x01 \x01(\tR\x04node\x12\x17\n" + diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index 92862c378..1ce00127c 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -439,7 +439,7 @@ func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_ser glog.V(0).Infof("VolumeEcShardsMount: %v", req) for _, shardId := range req.ShardIds { - err := vs.store.MountEcShards(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId)) + err := vs.store.MountEcShards(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId), req.SourceDiskType) if err != nil { glog.Errorf("ec shard mount %v: %v", req, err) diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index f0421f7f0..b5a966f2d 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -246,6 +246,24 @@ func (ev *EcVolume) Destroy() { os.Remove(ev.FileName(".vif")) } +// DiskType returns the disk type the EC volume currently reports under. +// Defaults to the physical location's disk type; orchestrators can override +// it via SetDiskType so the volume keeps reporting under the source +// volume's disk type after encoding (#9423). +func (ev *EcVolume) DiskType() types.DiskType { + return ev.diskType +} + +// SetDiskType overrides the EC volume's reported disk type and propagates +// to its mounted shards. Intended for the orchestrator-driven mount path +// (VolumeEcShardsMount); not persisted across restarts. +func (ev *EcVolume) SetDiskType(d types.DiskType) { + ev.diskType = d + for _, s := range ev.Shards { + s.DiskType = d + } +} + func (ev *EcVolume) FileName(ext string) string { switch ext { case ".ecx", ".ecj": diff --git a/weed/storage/erasure_coding/placement/placement.go b/weed/storage/erasure_coding/placement/placement.go index bda050b82..efa0afa26 100644 --- a/weed/storage/erasure_coding/placement/placement.go +++ b/weed/storage/erasure_coding/placement/placement.go @@ -10,6 +10,7 @@ package placement import ( "fmt" "sort" + "strings" ) // DiskCandidate represents a disk that can receive EC shards @@ -18,6 +19,7 @@ type DiskCandidate struct { DiskID uint32 DataCenter string Rack string + DiskType string // disk type (hdd/ssd/...) — empty means HardDrive // Capacity information VolumeCount int64 @@ -62,6 +64,13 @@ type PlacementRequest struct { // PreferDifferentRacks when true, spreads shards across different racks // before using multiple servers in the same rack PreferDifferentRacks bool + + // PreferredDiskType, when non-empty, biases placement toward disks of + // this type. Disks of the preferred type are exhausted (subject to the + // other diversity preferences) before disks of any other type are + // considered. Empty means no disk-type bias — all suitable disks form a + // single pool, matching pre-#9423 behavior. + PreferredDiskType string } // PlacementResult contains the selected destinations for EC shards @@ -77,12 +86,25 @@ type PlacementResult struct { ShardsPerServer map[string]int ShardsPerRack map[string]int ShardsPerDC map[string]int + + // SpilledToOtherDiskType is set when PlacementRequest.PreferredDiskType + // was non-empty but the preferred-type pool could not satisfy + // ShardsNeeded, so placement had to spill onto disks of other types. + // Callers can log a warning when this is true. + SpilledToOtherDiskType bool } // SelectDestinations selects the best disks for EC shard placement. // This is the main entry point for EC placement logic. // -// The algorithm works in multiple passes: +// Disk-type preference (#9423): when config.PreferredDiskType is non-empty, +// suitable disks are partitioned into a matching-type tier and a +// fallback tier. Each tier is run through the diversity passes below; +// the fallback tier is only consulted if the matching tier runs out of +// candidates before ShardsNeeded is satisfied. Empty PreferredDiskType +// processes all suitable disks as one tier, preserving prior behavior. +// +// Within each tier, the algorithm works in multiple passes: // 1. First pass: Select one disk from each rack (maximize rack diversity) // 2. Second pass: Select one disk from each unused server in used racks (maximize server diversity) // 3. Third pass: Select additional disks from servers already used (maximize disk diversity) @@ -100,9 +122,6 @@ func SelectDestinations(disks []*DiskCandidate, config PlacementRequest) (*Place return nil, fmt.Errorf("no suitable disks found after filtering") } - // Build indexes for efficient lookup - rackToDisks := groupDisksByRack(suitable) - result := &PlacementResult{ SelectedDisks: make([]*DiskCandidate, 0, config.ShardsNeeded), ShardsPerServer: make(map[string]int), @@ -114,7 +133,83 @@ func SelectDestinations(disks []*DiskCandidate, config PlacementRequest) (*Place usedServers := make(map[string]bool) // nodeID -> bool usedRacks := make(map[string]bool) // "dc:rack" -> bool - // Pass 1: Select one disk from each rack (maximize rack diversity) + // Partition suitable into preferred-disk-type / fallback tiers. + // Process the preferred tier first; only spill to fallback when the + // preferred pool can't satisfy ShardsNeeded. + preferredTier, fallbackTier := partitionByDiskType(suitable, config.PreferredDiskType) + selectFromTier(preferredTier, result, usedDisks, usedServers, usedRacks, config) + if config.PreferredDiskType != "" && len(result.SelectedDisks) < config.ShardsNeeded && len(fallbackTier) > 0 { + before := len(result.SelectedDisks) + selectFromTier(fallbackTier, result, usedDisks, usedServers, usedRacks, config) + if len(result.SelectedDisks) > before { + result.SpilledToOtherDiskType = true + } + } + + // Calculate final statistics + result.ServersUsed = len(usedServers) + result.RacksUsed = len(usedRacks) + dcSet := make(map[string]bool) + for _, disk := range result.SelectedDisks { + dcSet[disk.DataCenter] = true + } + result.DCsUsed = len(dcSet) + + return result, nil +} + +// partitionByDiskType splits disks into (matching, fallback) based on the +// preferred disk type. If preferred is empty, everything goes into the +// matching tier and fallback is empty — i.e. existing single-pool behavior. +// +// Empty DiskCandidate.DiskType is treated as HardDriveType ("hdd") to +// mirror weed/storage/types.ToDiskType's normalization, so a +// PreferredDiskType of "hdd" matches disks reporting "" — otherwise EC +// shards from an HDD source would always spill onto disks that happen to +// report their type as "" (HardDriveType). +func partitionByDiskType(disks []*DiskCandidate, preferred string) (matching, fallback []*DiskCandidate) { + if preferred == "" { + return disks, nil + } + pref := normalizeDiskType(preferred) + for _, d := range disks { + if normalizeDiskType(d.DiskType) == pref { + matching = append(matching, d) + } else { + fallback = append(fallback, d) + } + } + return matching, fallback +} + +// normalizeDiskType lower-cases the input and folds "" to "hdd" so the +// HardDriveType sentinel ("") and explicit "hdd"/"HDD" all compare equal. +func normalizeDiskType(t string) string { + t = strings.ToLower(t) + if t == "" { + return "hdd" + } + return t +} + +// selectFromTier runs the three diversity passes against `tier`, mutating +// `result` and the used* maps in place. Passes stop as soon as ShardsNeeded +// is reached. The function is a no-op when the tier is empty or the result +// already has enough shards, so it is safe to call once per tier. +func selectFromTier(tier []*DiskCandidate, result *PlacementResult, + usedDisks, usedServers, usedRacks map[string]bool, + config PlacementRequest) { + + if len(tier) == 0 || len(result.SelectedDisks) >= config.ShardsNeeded { + return + } + + rackToDisks := groupDisksByRack(tier) + + // Pass 1: Select one disk from each rack (maximize rack diversity). + // When this is the fallback tier (preferred tier already populated + // usedRacks), skip those racks so the spillover still spreads onto + // new racks instead of doubling up on ones already picked. if config.PreferDifferentRacks { // Sort racks by number of available servers (descending) to prioritize racks with more options sortedRacks := sortRacksByServerCount(rackToDisks) @@ -122,6 +217,9 @@ func SelectDestinations(disks []*DiskCandidate, config PlacementRequest) (*Place if len(result.SelectedDisks) >= config.ShardsNeeded { break } + if usedRacks[rackKey] { + continue + } rackDisks := rackToDisks[rackKey] // Select best disk from this rack, preferring a new server disk := selectBestDiskFromRack(rackDisks, usedServers, usedDisks, config) @@ -166,9 +264,9 @@ func SelectDestinations(disks []*DiskCandidate, config PlacementRequest) (*Place // Pass 3: Fill remaining slots from already-used servers (different disks) // Use round-robin across servers to balance shards evenly if len(result.SelectedDisks) < config.ShardsNeeded { - // Group remaining disks by server + // Group remaining disks by server (within this tier) serverToRemainingDisks := make(map[string][]*DiskCandidate) - for _, disk := range suitable { + for _, disk := range tier { if !usedDisks[getDiskKey(disk)] { serverToRemainingDisks[disk.NodeID] = append(serverToRemainingDisks[disk.NodeID], disk) } @@ -220,17 +318,6 @@ func SelectDestinations(disks []*DiskCandidate, config PlacementRequest) (*Place addDiskToResult(result, disk, usedDisks, usedServers, usedRacks) } } - - // Calculate final statistics - result.ServersUsed = len(usedServers) - result.RacksUsed = len(usedRacks) - dcSet := make(map[string]bool) - for _, disk := range result.SelectedDisks { - dcSet[disk.DataCenter] = true - } - result.DCsUsed = len(dcSet) - - return result, nil } // filterSuitableDisks filters disks that are suitable for EC placement diff --git a/weed/storage/erasure_coding/placement/placement_test.go b/weed/storage/erasure_coding/placement/placement_test.go new file mode 100644 index 000000000..6062d0ef3 --- /dev/null +++ b/weed/storage/erasure_coding/placement/placement_test.go @@ -0,0 +1,143 @@ +package placement + +import ( + "strconv" + "testing" +) + +// makeDisk builds a DiskCandidate with sensible defaults; tests override +// only the fields they care about. +func makeDisk(node, rack, diskType string, diskID uint32) *DiskCandidate { + return &DiskCandidate{ + NodeID: node, + DiskID: diskID, + DataCenter: "dc1", + Rack: rack, + DiskType: diskType, + VolumeCount: 0, + MaxVolumeCount: 100, + FreeSlots: 100, + } +} + +func disksByType(disks []*DiskCandidate) map[string]int { + out := map[string]int{} + for _, d := range disks { + out[d.DiskType]++ + } + return out +} + +func newRequest(shards int, preferred string) PlacementRequest { + return PlacementRequest{ + ShardsNeeded: shards, + PreferDifferentServers: true, + PreferDifferentRacks: true, + PreferredDiskType: preferred, + } +} + +// Plenty of SSD disks available: placement should fill entirely from SSD +// when PreferredDiskType="ssd", leaving HDDs untouched and not flagging +// spillover. +func TestSelectDestinations_PrefersMatchingDiskType(t *testing.T) { + var disks []*DiskCandidate + for i := 0; i < 6; i++ { + disks = append(disks, makeDisk("ssd-"+strconv.Itoa(i), "r"+strconv.Itoa(i%3), "ssd", uint32(i))) + } + for i := 0; i < 6; i++ { + disks = append(disks, makeDisk("hdd-"+strconv.Itoa(i), "r"+strconv.Itoa(i%3), "", uint32(i))) + } + + result, err := SelectDestinations(disks, newRequest(4, "ssd")) + if err != nil { + t.Fatalf("SelectDestinations: %v", err) + } + if got := len(result.SelectedDisks); got != 4 { + t.Fatalf("selected %d disks, want 4", got) + } + if counts := disksByType(result.SelectedDisks); counts["ssd"] != 4 { + t.Fatalf("disk-type counts = %v, want ssd=4", counts) + } + if result.SpilledToOtherDiskType { + t.Fatalf("SpilledToOtherDiskType should be false when preferred pool was sufficient") + } +} + +// Only one SSD disk available but 4 shards needed: placement must consume +// the SSD first, then spill to HDD for the remainder, and report spillover. +func TestSelectDestinations_SpillsWhenPreferredScarce(t *testing.T) { + disks := []*DiskCandidate{ + makeDisk("ssd-0", "r0", "ssd", 0), + makeDisk("hdd-0", "r1", "", 0), + makeDisk("hdd-1", "r2", "", 0), + makeDisk("hdd-2", "r3", "", 0), + } + + result, err := SelectDestinations(disks, newRequest(4, "ssd")) + if err != nil { + t.Fatalf("SelectDestinations: %v", err) + } + if got := len(result.SelectedDisks); got != 4 { + t.Fatalf("selected %d disks, want 4", got) + } + counts := disksByType(result.SelectedDisks) + if counts["ssd"] != 1 || counts[""] != 3 { + t.Fatalf("disk-type counts = %v, want ssd=1 hdd=3", counts) + } + if !result.SpilledToOtherDiskType { + t.Fatalf("SpilledToOtherDiskType should be true after falling back to HDD") + } +} + +// PreferredDiskType="hdd" must match disks whose DiskType is "" (the +// HardDriveType sentinel) — otherwise EC encoding of an HDD source would +// always spill onto HDDs that happen to report disk_type="" even though +// the cluster has plenty of matching capacity. +func TestSelectDestinations_PreferredHddMatchesEmptyDiskType(t *testing.T) { + disks := []*DiskCandidate{ + makeDisk("hdd-0", "r0", "", 0), // HardDriveType sentinel + makeDisk("hdd-1", "r1", "", 0), // HardDriveType sentinel + makeDisk("ssd-0", "r2", "ssd", 0), + } + + result, err := SelectDestinations(disks, newRequest(2, "hdd")) + if err != nil { + t.Fatalf("SelectDestinations: %v", err) + } + if got := len(result.SelectedDisks); got != 2 { + t.Fatalf("selected %d disks, want 2", got) + } + // Both selected disks must be HDD-reporting (i.e. DiskType == ""), + // and no spillover should have been required. + for _, d := range result.SelectedDisks { + if d.DiskType != "" { + t.Errorf("selected disk %s has DiskType=%q, want \"\" (HardDriveType)", d.NodeID, d.DiskType) + } + } + if result.SpilledToOtherDiskType { + t.Fatalf("SpilledToOtherDiskType should be false when HDD pool matches preferred=hdd") + } +} + +// Empty PreferredDiskType: pre-#9423 behavior, single pool, no spillover +// flag regardless of disk-type mix. +func TestSelectDestinations_EmptyPreferredDiskTypeKeepsPriorBehavior(t *testing.T) { + disks := []*DiskCandidate{ + makeDisk("ssd-0", "r0", "ssd", 0), + makeDisk("hdd-0", "r1", "", 0), + makeDisk("hdd-1", "r2", "", 0), + makeDisk("ssd-1", "r3", "ssd", 0), + } + + result, err := SelectDestinations(disks, newRequest(3, "")) + if err != nil { + t.Fatalf("SelectDestinations: %v", err) + } + if got := len(result.SelectedDisks); got != 3 { + t.Fatalf("selected %d disks, want 3", got) + } + if result.SpilledToOtherDiskType { + t.Fatalf("SpilledToOtherDiskType should never be set when PreferredDiskType is empty") + } +} diff --git a/weed/storage/erasure_coding/shard_distribution.go b/weed/storage/erasure_coding/shard_distribution.go index 6a8756bc8..591c60354 100644 --- a/weed/storage/erasure_coding/shard_distribution.go +++ b/weed/storage/erasure_coding/shard_distribution.go @@ -222,7 +222,9 @@ func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb } // MountEcShards mounts EC shards on destination servers using an assignment map. -func MountEcShards(volumeID uint32, collection string, shardAssignment map[string][]string, dialOption grpc.DialOption, logger logger) error { +// sourceDiskType is forwarded to VolumeEcShardsMount so the resulting EC volume +// reports under the source's disk type rather than the destination's (#9423). +func MountEcShards(volumeID uint32, collection string, shardAssignment map[string][]string, sourceDiskType string, dialOption grpc.DialOption, logger logger) error { if shardAssignment == nil { return fmt.Errorf("shard assignment not available for mounting") } @@ -263,9 +265,10 @@ func MountEcShards(volumeID uint32, collection string, shardAssignment map[strin err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), dialOption, func(client volume_server_pb.VolumeServerClient) error { _, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ - VolumeId: volumeID, - Collection: collection, - ShardIds: shardIds, + VolumeId: volumeID, + Collection: collection, + ShardIds: shardIds, + SourceDiskType: sourceDiskType, }) return mountErr }) diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index 00ec0b69e..a0157e353 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -159,11 +159,18 @@ func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat { } -func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId) error { +func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, sourceDiskType string) error { for diskId, location := range s.Locations { if ecVolume, err := location.LoadEcShard(collection, vid, shardId); err == nil { glog.V(0).Infof("MountEcShards %d.%d on disk ID %d", vid, shardId, diskId) + // Apply the orchestrator-supplied source disk type so the EC + // volume reports under it instead of the location's. Empty means + // "fall back to location's disk type" (#9423). + if sourceDiskType != "" { + ecVolume.SetDiskType(types.ToDiskType(sourceDiskType)) + } + si := erasure_coding.NewShardsInfo() si.Set(erasure_coding.NewShardInfo(shardId, erasure_coding.ShardSize(ecVolume.ShardSize()))) s.NewEcShardsChan <- master_pb.VolumeEcShardInformationMessage{ @@ -171,7 +178,7 @@ func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId er Collection: collection, EcIndexBits: uint32(si.Bitmap()), ShardSizes: si.SizesInt64(), - DiskType: string(location.DiskType), + DiskType: string(ecVolume.DiskType()), ExpireAtSec: ecVolume.ExpireAtSec, DiskId: uint32(diskId), } diff --git a/weed/storage/store_ec_disk_type_test.go b/weed/storage/store_ec_disk_type_test.go new file mode 100644 index 000000000..6695d5876 --- /dev/null +++ b/weed/storage/store_ec_disk_type_test.go @@ -0,0 +1,212 @@ +package storage + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/storage/volume_info" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// setupECStoreWithMixedDisks builds an empty two-location Store +// (HDD + SSD) and starts a goroutine that captures every +// NewEcShardsChan message published after setup. The caller is expected +// to drop an EC shard file plus matching .ecx/.ecj/.vif onto one of the +// locations and then invoke MountEcShards — this mirrors the real +// VolumeEcShardsCopy → VolumeEcShardsMount sequence and avoids racing +// with the startup-scan publish (which uses the location's disk type +// and would otherwise pollute the captured slice). The returned slice +// is appended to only by the goroutine; tests should not race against +// it directly — use waitForShardMsg instead. +func setupECStoreWithMixedDisks(t *testing.T) (store *Store, drainedShardMsgs *[]master_pb.VolumeEcShardInformationMessage, vid needle.VolumeId, collection string, plant func()) { + t.Helper() + tempDir := t.TempDir() + hddDir := filepath.Join(tempDir, "hdd") + ssdDir := filepath.Join(tempDir, "ssd") + for _, d := range []string{hddDir, ssdDir} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + + collection = "logs" + vid = needle.VolumeId(4242) + const dataShards, parityShards = 10, 4 + const datSize int64 = 10 * 1024 * 1024 + expectedShardSize := calculateExpectedShardSize(datSize, dataShards) + + store = NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{hddDir, ssdDir}, + []int32{100, 100}, + []util.MinFreeSpace{{}, {}}, + "", + NeedleMapInMemory, + []types.DiskType{types.HardDriveType, types.SsdType}, + nil, + 3, + ) + + captured := []master_pb.VolumeEcShardInformationMessage{} + drainedShardMsgs = &captured + done := make(chan struct{}) + go func() { + for { + select { + case msg := <-store.NewEcShardsChan: + captured = append(captured, msg) + case <-store.NewVolumesChan: + case <-store.DeletedVolumesChan: + case <-store.DeletedEcShardsChan: + case <-store.StateUpdateChan: + case <-done: + return + } + } + }() + t.Cleanup(func() { + store.Close() + close(done) + }) + + // Caller invokes plant() after the store exists; this mirrors a + // VolumeEcShardsCopy delivering shard + index files onto the HDD + // location before a subsequent VolumeEcShardsMount. + plant = func() { + t.Helper() + base := erasure_coding.EcShardFileName(collection, hddDir, int(vid)) + f, err := os.Create(base + erasure_coding.ToExt(0)) + if err != nil { + t.Fatalf("create shard: %v", err) + } + if err := f.Truncate(expectedShardSize); err != nil { + f.Close() + t.Fatalf("truncate shard: %v", err) + } + f.Close() + if err := os.WriteFile(base+".ecx", make([]byte, 20), 0o644); err != nil { + t.Fatalf("write .ecx: %v", err) + } + if err := os.WriteFile(base+".ecj", nil, 0o644); err != nil { + t.Fatalf("write .ecj: %v", err) + } + if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + DatFileSize: datSize, + EcShardConfig: &volume_server_pb.EcShardConfig{ + DataShards: dataShards, + ParityShards: parityShards, + }, + }); err != nil { + t.Fatalf("save .vif: %v", err) + } + } + + return store, drainedShardMsgs, vid, collection, plant +} + +// waitForShardMsg returns the first captured mount message for vid, polling +// briefly because the goroutine reads asynchronously from MountEcShards. +func waitForShardMsg(t *testing.T, captured *[]master_pb.VolumeEcShardInformationMessage, vid needle.VolumeId) master_pb.VolumeEcShardInformationMessage { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + for _, msg := range *captured { + if msg.Id == uint32(vid) { + return msg + } + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("no NewEcShardsChan message captured for volume %d within 2s", vid) + return master_pb.VolumeEcShardInformationMessage{} +} + +// findHeartbeatShard returns the EC-shard heartbeat entry for vid. +func findHeartbeatShard(t *testing.T, hb *master_pb.Heartbeat, vid needle.VolumeId) *master_pb.VolumeEcShardInformationMessage { + t.Helper() + for _, msg := range hb.EcShards { + if msg.Id == uint32(vid) { + return msg + } + } + t.Fatalf("no heartbeat EcShards entry for volume %d", vid) + return nil +} + +// TestMountEcShards_AppliesSourceDiskType covers the #9423 mount path: a +// shard physically lives on an HDD location, but MountEcShards is told the +// source volume's disk type was "ssd". The volume server must surface that +// "ssd" value on (a) the one-shot NewEcShardsChan notification, (b) the +// in-memory EcVolume, (c) the mounted shard, and (d) the steady-state +// CollectErasureCodingHeartbeat output. +func TestMountEcShards_AppliesSourceDiskType(t *testing.T) { + store, captured, vid, collection, plant := setupECStoreWithMixedDisks(t) + plant() + + if err := store.MountEcShards(collection, vid, 0, "ssd"); err != nil { + t.Fatalf("MountEcShards: %v", err) + } + + mountMsg := waitForShardMsg(t, captured, vid) + if mountMsg.DiskType != "ssd" { + t.Errorf("NewEcShardsChan disk_type = %q, want %q", mountMsg.DiskType, "ssd") + } + + hb := store.CollectErasureCodingHeartbeat() + heartbeatMsg := findHeartbeatShard(t, hb, vid) + if heartbeatMsg.DiskType != "ssd" { + t.Errorf("heartbeat EcShards disk_type = %q, want %q", heartbeatMsg.DiskType, "ssd") + } + + // Inspect the in-memory state directly: both the EcVolume and the shard + // it holds must report the overridden disk type so subsequent + // unmount/delete messages stay consistent with mount. + loc := store.Locations[0] // HDD location holds the shard + ecVol, found := loc.FindEcVolume(vid) + if !found { + t.Fatalf("EC volume %d not found on HDD location after mount", vid) + } + if got := string(ecVol.DiskType()); got != "ssd" { + t.Errorf("EcVolume.DiskType() = %q, want %q", got, "ssd") + } + shard, ok := ecVol.FindEcVolumeShard(0) + if !ok { + t.Fatalf("EcVolume %d has no shard 0 mounted", vid) + } + if got := string(shard.DiskType); got != "ssd" { + t.Errorf("EcVolumeShard.DiskType = %q, want %q", got, "ssd") + } +} + +// TestMountEcShards_FallsBackToLocationDiskTypeWhenEmpty pins the +// pre-#9423 behavior for the empty-source path: disk-scan reload, the +// orphan-shard reconciler, and any other call site that has no +// orchestrator context passes "" and must still get the location's disk +// type on the heartbeat — i.e. the override path must not regress them. +func TestMountEcShards_FallsBackToLocationDiskTypeWhenEmpty(t *testing.T) { + store, captured, vid, collection, plant := setupECStoreWithMixedDisks(t) + plant() + + if err := store.MountEcShards(collection, vid, 0, ""); err != nil { + t.Fatalf("MountEcShards: %v", err) + } + + mountMsg := waitForShardMsg(t, captured, vid) + // HDD location -> HardDriveType -> empty disk_type string on the wire. + if mountMsg.DiskType != string(types.HardDriveType) { + t.Errorf("NewEcShardsChan disk_type = %q, want %q (location's HDD)", mountMsg.DiskType, string(types.HardDriveType)) + } + + hb := store.CollectErasureCodingHeartbeat() + heartbeatMsg := findHeartbeatShard(t, hb, vid) + if heartbeatMsg.DiskType != string(types.HardDriveType) { + t.Errorf("heartbeat disk_type = %q, want %q", heartbeatMsg.DiskType, string(types.HardDriveType)) + } +} diff --git a/weed/worker/tasks/erasure_coding/detection.go b/weed/worker/tasks/erasure_coding/detection.go index 6d63738f4..a2c045ff8 100644 --- a/weed/worker/tasks/erasure_coding/detection.go +++ b/weed/worker/tasks/erasure_coding/detection.go @@ -286,7 +286,7 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste Targets: createECTargets(multiPlan, dataShards, parityShards), TaskParams: &worker_pb.TaskParams_ErasureCodingParams{ - ErasureCodingParams: createECTaskParams(dataShards, parityShards), + ErasureCodingParams: createECTaskParams(dataShards, parityShards, metric.DiskType), }, } @@ -391,7 +391,7 @@ func newECPlacementPlanner(activeTopology *topology.ActiveTopology, preferredTag } } -func (p *ecPlacementPlanner) selectDestinations(sourceRack, sourceDC string, shardsNeeded int) ([]*placement.DiskCandidate, error) { +func (p *ecPlacementPlanner) selectDestinations(sourceRack, sourceDC, sourceDiskType string, shardsNeeded int) ([]*placement.DiskCandidate, error) { if p == nil || p.activeTopology == nil { return nil, fmt.Errorf("ec placement planner is not initialized") } @@ -406,6 +406,10 @@ func (p *ecPlacementPlanner) selectDestinations(sourceRack, sourceDC string, sha MaxTaskLoad: topology.MaxTaskLoadForECPlacement, PreferDifferentServers: true, PreferDifferentRacks: true, + // Bias placement toward disks matching the source volume's disk + // type; placement spills to other types only if the preferred + // pool can't satisfy ShardsNeeded (#9423). + PreferredDiskType: sourceDiskType, } var lastErr error @@ -415,6 +419,10 @@ func (p *ecPlacementPlanner) selectDestinations(sourceRack, sourceDC string, sha } result, err := placement.SelectDestinations(candidates, config) if err == nil { + if result.SpilledToOtherDiskType { + glog.Warningf("EC placement spilled to disks outside preferred disk type %q to reach %d shards (source rack=%s dc=%s)", + sourceDiskType, shardsNeeded, sourceRack, sourceDC) + } return result.SelectedDisks, nil } lastErr = err @@ -633,8 +641,9 @@ func planECDestinations(planner *ecPlacementPlanner, metric *types.VolumeHealthM } } - // Select best disks for EC placement with rack/DC diversity using the cached planner - selectedDisks, err := planner.selectDestinations(sourceRack, sourceDC, totalShards) + // Select best disks for EC placement with rack/DC diversity using the cached planner. + // Pass source disk type so placement prefers matching-type disks (#9423). + selectedDisks, err := planner.selectDestinations(sourceRack, sourceDC, metric.DiskType, totalShards) if err != nil { return nil, err } @@ -786,11 +795,14 @@ func convertTaskSourcesToProtobuf(sources []topology.TaskSourceSpec, volumeID ui return protobufSources, nil } -// createECTaskParams creates clean EC task parameters (destinations now in unified targets) -func createECTaskParams(dataShards, parityShards int) *worker_pb.ErasureCodingTaskParams { +// createECTaskParams creates clean EC task parameters (destinations now in unified targets). +// sourceDiskType is forwarded to VolumeEcShardsMount so the resulting EC volume +// reports under the source's disk type rather than the target location's (#9423). +func createECTaskParams(dataShards, parityShards int, sourceDiskType string) *worker_pb.ErasureCodingTaskParams { return &worker_pb.ErasureCodingTaskParams{ - DataShards: int32(dataShards), - ParityShards: int32(parityShards), + DataShards: int32(dataShards), + ParityShards: int32(parityShards), + SourceDiskType: sourceDiskType, } } @@ -824,6 +836,7 @@ func diskInfosToCandidates(disks []*topology.DiskInfo) []*placement.DiskCandidat DiskID: disk.DiskID, DataCenter: disk.DataCenter, Rack: disk.Rack, + DiskType: disk.DiskType, VolumeCount: disk.DiskInfo.VolumeCount, MaxVolumeCount: disk.DiskInfo.MaxVolumeCount, ShardCount: ecShardCount, diff --git a/weed/worker/tasks/erasure_coding/detection_disk_type_test.go b/weed/worker/tasks/erasure_coding/detection_disk_type_test.go new file mode 100644 index 000000000..efee93aca --- /dev/null +++ b/weed/worker/tasks/erasure_coding/detection_disk_type_test.go @@ -0,0 +1,94 @@ +package erasure_coding + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/worker/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestPlanECDestinationsPrefersSourceDiskType_FullCluster covers the +// #9423 happy path through the worker plumbing: every node carries both +// an HDD and an SSD disk, so the cluster has enough SSD slots to fully +// satisfy a 10+4 placement. With metric.DiskType="ssd", every selected +// destination must land on an SSD disk. +func TestPlanECDestinationsPrefersSourceDiskType_FullCluster(t *testing.T) { + // 14 nodes × 2 disks per node (hdd + ssd) — enough SSD slots alone + // for a 10+4 layout with one-shard-per-(server,disk) diversity. + activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0) + + planner := newECPlacementPlanner(activeTopology, nil) + require.NotNil(t, planner) + + metric := &types.VolumeHealthMetrics{ + VolumeID: 1, + Server: "10.0.0.1:8080", + Size: 100 * 1024 * 1024, + Collection: "", + DiskType: "ssd", // the property being plumbed end-to-end + } + + plan, err := planECDestinations(planner, metric, NewDefaultConfig(), erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) + require.NoError(t, err) + require.Len(t, plan.Plans, erasure_coding.TotalShardsCount) + + // buildActiveTopology assigns DiskID by index into the diskTypes + // slice, so SSD is DiskID 1. Every plan entry must point at an SSD + // disk — anything else means metric.DiskType was dropped on the way + // to the placement layer. + for _, p := range plan.Plans { + assert.Equalf(t, uint32(1), p.TargetDisk, + "target %s diskID = %d, want SSD disk (DiskID=1)", p.TargetNode, p.TargetDisk) + } +} + +// TestPlanECDestinationsSpillsToOtherDiskType_WhenPreferredScarce pins +// the prefer-with-spillover policy: only one node in the cluster has an +// SSD disk; the rest are HDD-only. A 10+4 placement must still succeed +// (no hard fail), filling the one SSD slot first and the remaining 13 +// from HDD nodes. Without the spillover behavior this would either +// error out or starve the placement. +func TestPlanECDestinationsSpillsToOtherDiskType_WhenPreferredScarce(t *testing.T) { + // Start with every node carrying both HDD and SSD, then strip SSD + // from all but the first node so the SSD pool is too small alone. + activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0) + topo := activeTopology.GetTopologyInfo() + for _, dc := range topo.DataCenterInfos { + for _, rack := range dc.RackInfos { + for i, node := range rack.DataNodeInfos { + if i == 0 { + continue // keep the first node's SSD + } + delete(node.DiskInfos, "ssd") + } + } + } + require.NoError(t, activeTopology.UpdateTopology(topo)) + + planner := newECPlacementPlanner(activeTopology, nil) + require.NotNil(t, planner) + + metric := &types.VolumeHealthMetrics{ + VolumeID: 2, + Server: "10.0.0.1:8080", + Size: 100 * 1024 * 1024, + Collection: "", + DiskType: "ssd", + } + + plan, err := planECDestinations(planner, metric, NewDefaultConfig(), erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) + require.NoError(t, err) + require.Len(t, plan.Plans, erasure_coding.TotalShardsCount) + + // Exactly one SSD target (the only SSD node), the rest on HDD. + ssdHits := 0 + for _, p := range plan.Plans { + if p.TargetDisk == 1 { + ssdHits++ + } + } + assert.Equalf(t, 1, ssdHits, + "expected exactly 1 SSD placement (the only SSD node in the cluster), got %d", ssdHits) +} diff --git a/weed/worker/tasks/erasure_coding/detection_test.go b/weed/worker/tasks/erasure_coding/detection_test.go index 1f51b8801..4ded8e2bc 100644 --- a/weed/worker/tasks/erasure_coding/detection_test.go +++ b/weed/worker/tasks/erasure_coding/detection_test.go @@ -84,7 +84,7 @@ func TestECPlacementPlannerPrefersTaggedDisks(t *testing.T) { planner := newECPlacementPlanner(activeTopology, []string{"fast"}) require.NotNil(t, planner) - selected, err := planner.selectDestinations("", "", 2) + selected, err := planner.selectDestinations("", "", "", 2) require.NoError(t, err) require.Len(t, selected, 2) @@ -113,7 +113,7 @@ func TestECPlacementPlannerFallsBackWhenTagsInsufficient(t *testing.T) { planner := newECPlacementPlanner(activeTopology, []string{"fast"}) require.NotNil(t, planner) - selected, err := planner.selectDestinations("", "", 3) + selected, err := planner.selectDestinations("", "", "", 3) require.NoError(t, err) require.Len(t, selected, 3) diff --git a/weed/worker/tasks/erasure_coding/ec_task.go b/weed/worker/tasks/erasure_coding/ec_task.go index 0a49e3ac1..da537b2a5 100644 --- a/weed/worker/tasks/erasure_coding/ec_task.go +++ b/weed/worker/tasks/erasure_coding/ec_task.go @@ -37,6 +37,7 @@ type ErasureCodingTask struct { // EC parameters dataShards int32 parityShards int32 + sourceDiskType string // source volume's disk type, forwarded to Mount RPC (#9423) targets []*worker_pb.TaskTarget // Unified targets for EC shards sources []*worker_pb.TaskSource // Unified sources for cleanup shardAssignment map[string][]string // destination -> assigned shard types @@ -68,6 +69,7 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP t.dataShards = ecParams.DataShards t.parityShards = ecParams.ParityShards + t.sourceDiskType = ecParams.SourceDiskType t.workDir = ecParams.WorkingDir t.targets = params.Targets // Get unified targets t.sources = params.Sources // Get unified sources @@ -540,7 +542,7 @@ func (t *ErasureCodingTask) distributeEcShards(shardFiles map[string]string) err // mountEcShards mounts EC shards on destination servers func (t *ErasureCodingTask) mountEcShards() error { - return erasure_coding.MountEcShards(t.volumeID, t.collection, t.shardAssignment, t.grpcDialOption, t.GetLogger()) + return erasure_coding.MountEcShards(t.volumeID, t.collection, t.shardAssignment, t.sourceDiskType, t.grpcDialOption, t.GetLogger()) } // deleteOriginalVolume deletes the original volume and all its replicas from all servers diff --git a/weed/worker/tasks/erasure_coding/plugin_handler.go b/weed/worker/tasks/erasure_coding/plugin_handler.go index 123829c82..ba22b010a 100644 --- a/weed/worker/tasks/erasure_coding/plugin_handler.go +++ b/weed/worker/tasks/erasure_coding/plugin_handler.go @@ -713,6 +713,7 @@ func decodeErasureCodingTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParam if parityShards <= 0 { parityShards = int32(ecstorage.ParityShardsCount) } + sourceDiskType := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_disk_type", "")) totalShards := int(dataShards + parityShards) if volumeID == 0 { @@ -758,8 +759,9 @@ func decodeErasureCodingTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParam Targets: targets, TaskParams: &worker_pb.TaskParams_ErasureCodingParams{ ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{ - DataShards: dataShards, - ParityShards: parityShards, + DataShards: dataShards, + ParityShards: parityShards, + SourceDiskType: sourceDiskType, }, }, }, nil