mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-13 03:24:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(shell): don't halt volume.fsck purge on a stuck read-only volume (#9714)

Browse Source 
* fix(shell): don't halt volume.fsck purge on a stuck read-only volume

A failed VolumeMarkWritable on one volume aborted the entire fsck purge
run; per-volume errors now log and continue so remaining volumes still
get purged.

* fix(shell): unify volume.fsck per-volume skip logging at the caller

Return the mark-writable error from purgeOneVolume instead of logging
in two places — the caller already prints "skip purging volume N: %v"
and defers still fire on the error return.

* fix(shell): collect volume.fsck purge-skipped volumes and report at end

Track volume IDs whose purge was skipped (mark-writable failure or
other per-volume errors) and print a sorted summary so operators don't
have to scrape the run log to find them. Deletes for those volumes are
already skipped; this just makes them explicit.
This commit is contained in:
Chris Lu
2026-05-27 17:49:35 -07:00
committed by GitHub
parent bbbc3925ec
commit 24e664d651
1 changed files with 13 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/shell/command_volume_fsck.go
+13 -3
View File
@@ -448,6 +448,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn
// MasterClient.GetLocations, so iterating per replica here (as the old
// code did) would issue N*N delete RPCs for N replicas.
if applyPurging {
var skippedVolumeIds []uint32
for volumeId, orphanReplicaFileIds := range volumeIdOrphanFileIds {
if len(orphanReplicaFileIds) == 0 {
continue
@@ -458,11 +459,19 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn
}
// Call out to a closure per volume so the deferred "mark
// readonly again" fires between volumes instead of piling up
// until findExtraChunksInVolumeServers returns.
// until findExtraChunksInVolumeServers returns. Per-volume
// failures (e.g. a replica stuck read-only) don't halt the
// rest of the run; the volume is remembered and its deletes
// are skipped.
if err := c.purgeOneVolume(volumeId, orphanReplicaFileIds, volumeReplicaCounts[volumeId], readOnlyServerReplicas[volumeId]); err != nil {
return err
fmt.Fprintf(c.writer, "skip purging volume %d: %v\n", volumeId, err)
skippedVolumeIds = append(skippedVolumeIds, volumeId)
}
}
if len(skippedVolumeIds) > 0 {
sort.Slice(skippedVolumeIds, func(i, j int) bool { return skippedVolumeIds[i] < skippedVolumeIds[j] })
fmt.Fprintf(c.writer, "skipped purge on %d volume(s): %v\n", len(skippedVolumeIds), skippedVolumeIds)
}
}
if !applyPurging {
@@ -510,7 +519,8 @@ func (c *commandVolumeFsck) purgeOneVolume(volumeId uint32, orphanReplicaFileIds
needleVID := needle.VolumeId(volumeId)
for _, server := range readOnlyReplicas {
if err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, server, true, false); err != nil {
return fmt.Errorf("mark volume %d on %v read/write: %v", volumeId, server, err)
// Replicas flipped writable earlier roll back via the defer.
return fmt.Errorf("mark %v writable: %v", server, err)
}
fmt.Fprintf(c.writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, server)
defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, server, false, false)