ec.encode: count shards wherever they landed before deleting the source (#10483)

generateEcShards writes shards beside the source volume, so encoding a
volume that lives on a non-default medium puts them on that medium while
-diskType still says hdd. The pre-delete check counted only the -diskType
bucket, so it saw a complete set as zero shards, called it unrecoverable
and aborted -- leaving the volume as both a .dat and a full shard set,
which every later reader then disagrees about.

Count by node across disks, as waitForEcShardsToRegister in the same file
already does. The spread check is unaffected: it locates shards through
collectEcShardBitsByNode and only uses diskType to find free slots.

weed/shell/command_ec_encode.go

@@ -830,19 +830,26 @@ func verifyEcShardsBeforeDelete(commandEnv *CommandEnv, volumeIds []needle.Volum
 		lastDegraded = lastDegraded[:0]
 		lastClumped = lastClumped[:0]
 		for _, vid := range volumeIds {
-			nodeShards, _ := collectEcNodeShardsInfo(topoInfo, vid, diskType)
+			// Count the shards wherever they landed, as waitForEcShardsToRegister
+			// above already does. generateEcShards writes them beside the source
+			// volume, so encoding a volume that lives on a non-default medium
+			// puts them on that medium while -diskType still says hdd. Counting
+			// only the -diskType bucket then reports a complete set as entirely
+			// missing and aborts an encode that in fact succeeded, leaving the
+			// volume as both a .dat and a full set of shards.
+			byNode := collectEcShardBitsByNode(topoInfo, vid)
 
 			var union erasure_coding.ShardBits
-			for _, info := range nodeShards {
-				union = erasure_coding.ShardBits(uint32(union) | info.Bitmap())
+			for _, bits := range byNode {
+				union |= bits
 			}
 
 			totalShards := erasure_coding.TotalShardsCount
 			degraded, err := erasure_coding.RequireRecoverableShardSet(uint32(vid), union, erasure_coding.DataShardsCount, totalShards)
 			if err != nil {
-				summary := make([]string, 0, len(nodeShards))
-				for node, info := range nodeShards {
-					summary = append(summary, fmt.Sprintf("%s=%s", node, info.String()))
+				summary := make([]string, 0, len(byNode))
+				for node, bits := range byNode {
+					summary = append(summary, fmt.Sprintf("%s=%d shards", node, bits.Count()))
 				}
 				sort.Strings(summary)
 				lastErr = fmt.Errorf("volume %d: %w (observed: %v)", vid, err, summary)
@@ -859,8 +866,8 @@ func verifyEcShardsBeforeDelete(commandEnv *CommandEnv, volumeIds []needle.Volum
 				continue
 			}
 
-			glog.V(0).Infof("EC shard verification ok for volume %d on diskType %q: %d/%d shards present across %d nodes",
-				vid, diskType.ReadableString(), union.Count(), totalShards, len(nodeShards))
+			glog.V(0).Infof("EC shard verification ok for volume %d: %d/%d shards present across %d nodes",
+				vid, union.Count(), totalShards, len(byNode))
 		}
 
 		if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 {

weed/shell/command_ec_encode_test.go

@@ -433,3 +433,36 @@ func TestEcShardsClumpedOnOneNode(t *testing.T) {
 	assert.True(t, clumped)
 	assert.Equal(t, pb.NewServerAddressFromDataNode(fresh), holder)
 }
+
+// A volume that lived on ssd gets its shards written beside it, on ssd, while
+// -diskType still defaults to hdd. The pre-delete check must count them anyway:
+// scoping the count to the hdd bucket saw a complete set as zero shards and
+// aborted the encode, leaving the volume as both a .dat and a full shard set.
+func TestEcShardCountIgnoresDiskTypeOfTheShards(t *testing.T) {
+	allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
+	node := &master_pb.DataNodeInfo{
+		Id: "node1:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"":    {MaxVolumeCount: 10},
+			"ssd": {Type: "ssd", MaxVolumeCount: 10, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{Id: 1, EcIndexBits: allBits, DiskId: 1}}},
+		},
+	}
+	topo := ecShardVisibilityTestTopology(node)
+
+	var union erasure_coding.ShardBits
+	for _, bits := range collectEcShardBitsByNode(topo, needle.VolumeId(1)) {
+		union |= bits
+	}
+	assert.Equal(t, erasure_coding.TotalShardsCount, union.Count(),
+		"shards on a non-default medium must still be counted")
+
+	degraded, err := erasure_coding.RequireRecoverableShardSet(
+		1, union, erasure_coding.DataShardsCount, erasure_coding.TotalShardsCount)
+	assert.NoError(t, err, "a complete shard set must not read as unrecoverable")
+	assert.False(t, degraded)
+
+	// The disk-scoped view is what the check used to consult, and it is blind
+	// to this volume entirely.
+	scoped, _ := collectEcNodeShardsInfo(topo, needle.VolumeId(1), types.ToDiskType(""))
+	assert.Empty(t, scoped, "the hdd-scoped view cannot see ssd shards")
+}