diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index fd3155197..94c0a651d 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -45,18 +46,19 @@ const ( ) type commandVolumeFsck struct { - env *CommandEnv - writer io.Writer - bucketsPath string - collection *string - volumeIds map[uint32]bool - tempFolder string - verbose *bool - forcePurging *bool - skipEcVolumes *bool - findMissingChunksInFiler *bool - verifyNeedle *bool - filerSigningKey string + env *CommandEnv + writer io.Writer + bucketsPath string + collection *string + volumeIds map[uint32]bool + tempFolder string + verbose *bool + forcePurging *bool + skipEcVolumes *bool + findMissingChunksInFiler *bool + verifyNeedle *bool + filerSigningKey string + unresolvedManifestEntries atomic.Int64 } func (c *commandVolumeFsck) Name() string { @@ -113,6 +115,12 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } + // The command struct is a singleton registered in init(), so any state + // not bound to a flag persists across shell invocations. Reset the + // unresolved-manifest counter so a previous failed run can't permanently + // suppress -reallyDeleteFromVolume in this session. + c.unresolvedManifestEntries.Store(0) + if err = commandEnv.confirmIsLocked(args); err != nil { return } @@ -219,8 +227,19 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, false, 0, 0); err != nil { return fmt.Errorf("failed to collect file ids from filer: %w", err) } + // If any entry's manifest could not be resolved, our in-use fid set + // is missing the sub-chunks behind it. Purging orphans now would + // delete live data referenced only via the unresolved manifest, so + // disable -reallyDeleteFromVolume for this run and tell the operator + // to fix the broken entries first. + applyPurgingEffective := *applyPurging + if unresolved := c.unresolvedManifestEntries.Load(); unresolved > 0 && applyPurgingEffective { + fmt.Fprintf(c.writer, "WARNING: %d entry(ies) had unresolvable chunk manifests; refusing to apply -reallyDeleteFromVolume to avoid deleting live sub-chunks. Fix the entries listed above (e.g. delete or repair them) and re-run.\n", + unresolved) + applyPurgingEffective = false + } // volume file ids subtract filer file ids - if err = c.findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo, *applyPurging, uint64(collectModifyFromAtNs), uint64(collectCutoffFromAtNs)); err != nil { + if err = c.findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo, applyPurgingEffective, uint64(collectModifyFromAtNs), uint64(collectCutoffFromAtNs)); err != nil { return fmt.Errorf("findExtraChunksInVolumeServers: %w", err) } } @@ -257,9 +276,28 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m if *c.verbose && entry.Entry.IsDirectory { fmt.Fprintf(c.writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name)) } - dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(context.Background(), filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64) + dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(ctx, filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64) if resolveErr != nil { - return fmt.Errorf("failed to ResolveChunkManifest: %+v", resolveErr) + // Cancellation/deadline isn't manifest corruption; surface it + // so the BFS bails out cleanly without polluting the + // unresolved-manifest counter (which would otherwise block + // purges and mislead the operator about the failure cause). + if errors.Is(resolveErr, context.Canceled) || errors.Is(resolveErr, context.DeadlineExceeded) { + return resolveErr + } + // A single broken manifest used to abort the whole traversal, + // leaving the operator with no way to identify orphans without + // first fixing the broken file. Instead, record only the + // top-level chunk fids (data chunks plus the manifest needles + // themselves — sub-chunks behind the unreadable manifest are + // unknown), warn, and keep going. The unresolved counter blocks + // any purge step downstream so we never delete a sub-chunk we + // couldn't account for. + fmt.Fprintf(c.writer, "WARNING: ResolveChunkManifest failed for %s: %v — recording top-level chunk fids only; purging will be disabled\n", + util.NewFullPath(entry.Dir, entry.Entry.Name), resolveErr) + c.unresolvedManifestEntries.Add(1) + dataChunks = entry.Entry.GetChunks() + manifestChunks = nil } dataChunks = append(dataChunks, manifestChunks...) for _, chunk := range dataChunks {