From cac83bb4a80c6a549b6078a97ed91ace644607eb Mon Sep 17 00:00:00 2001 From: Lisandro Pin Date: Tue, 30 Jun 2026 11:05:14 +0200 Subject: [PATCH] Fix scrubbing of deleted needles on EC volumes. (#10130) EC volumes do not propagate deletions to all shard indexes, so it is possible to run scrubbing on a volume where a deleted needle is still present in the index, or a needle deleted from the index is still present on the volume. On either scenario, scrubbing will fail due to size mismatch errors. This PR reworks the scrubbing logic so needle size mismatches are ignored in such scenarios. Scrubbing can still be forced to check deleted needles (f.ex. to discover index inconsistencies); this option will be exposed in RPCs and `weed shell` on a follow-up PR. --- weed/server/volume_grpc_scrub.go | 3 ++- weed/storage/store_ec_scrub.go | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/weed/server/volume_grpc_scrub.go b/weed/server/volume_grpc_scrub.go index c306944ae..3b967e35c 100644 --- a/weed/server/volume_grpc_scrub.go +++ b/weed/server/volume_grpc_scrub.go @@ -124,7 +124,8 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb case volume_server_pb.VolumeScrubMode_LOCAL: files, shardInfos, serrs = v.ScrubLocal() case volume_server_pb.VolumeScrubMode_FULL: - files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId) + // TODO: expose force_deleted_needles_check in the RPC and weed shell. + files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId, false) case volume_server_pb.VolumeScrubMode_CHECKSUM: // Verify each local shard's raw bytes against the bitrot sidecar, // exercising cold parity shards. Read-only. ChecksumScrub's first diff --git a/weed/storage/store_ec_scrub.go b/weed/storage/store_ec_scrub.go index fbd9eccce..2de3da1c4 100644 --- a/weed/storage/store_ec_scrub.go +++ b/weed/storage/store_ec_scrub.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "fmt" "slices" @@ -12,7 +13,7 @@ import ( // ScrubEcVolume checks the full integrity of a EC volume, across both local and remote shards. // Returns a count of processed file entries, slice of found broken shards, and slice of found errors. -func (s *Store) ScrubEcVolume(vid needle.VolumeId) (int64, []*volume_server_pb.EcShardInfo, []error) { +func (s *Store) ScrubEcVolume(vid needle.VolumeId, forceDeletedNeedlesCheck bool) (int64, []*volume_server_pb.EcShardInfo, []error) { ecv, found := s.FindEcVolume(vid) if !found { return 0, nil, []error{fmt.Errorf("EC volume id %d not found", vid)} @@ -78,7 +79,12 @@ func (s *Store) ScrubEcVolume(vid needle.VolumeId) (int64, []*volume_server_pb.E n := needle.Needle{} if err := n.ReadBytes(data, 0, size, ecv.Version); err != nil { - errs = append(errs, fmt.Errorf("needle %d on EC volume %d: %v", id, ecv.VolumeId, err)) + // needles flagged as deleted in the index but not in the volume (or vice-versa) cannot + // be properly hydrated, as the header read by needle.ReadBytes() will mismatch. + deleteSizeMismatch := size.IsDeleted() != (n.Size == 0) + if !errors.Is(err, needle.ErrorSizeMismatch) || !deleteSizeMismatch || forceDeletedNeedlesCheck { + errs = append(errs, fmt.Errorf("needle %d on EC volume %d: %v", id, ecv.VolumeId, err)) + } } return nil