seaweedfs/weed/command/fix.go

package command

import (
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage"
	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

func init() {
	cmdFix.Run = runFix // break init cycle
}

var cmdFix = &Command{
	UsageLine: "fix [-remoteFile=false] [-volumeId=234] [-collection=bigData] /tmp",
	Short:     "run weed tool fix on files or whole folders to recreate index file(s) if corrupted",
	Long: `Fix runs the SeaweedFS fix command on local dat files ( or remote files) or whole folders to re-create the index .idx file. If fixing remote files, you need to synchronize master.toml to the same directory on the current node as on the master node.
  You Need to stop the volume server when running this command.
  Use -ecx to rebuild a lost EC index (.ecx) — and the .vif when missing — from the local .ec## shards.
`,
}

var (
	fixVolumeCollection      = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
	fixVolumeId              = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
	fixIncludeDeleted        = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
	fixIgnoreError           = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
	fixRemoteFile            = cmdFix.Flag.Bool("remoteFile", false, "an optional, if true will not try to load the local .dat file, but only the remote file")
	fixGenerateEcx           = cmdFix.Flag.Bool("ecx", false, "regenerate a lost EC index (.ecx) — and the .vif when missing — from the local .ec## shards (missing shards are reconstructed from parity when enough survive). Run with the volume server stopped.")
	fixEcDataShards          = cmdFix.Flag.Int("ecDataShards", 0, "EC data shard count for -ecx (0 = read from .vif, otherwise default 10)")
	fixEcParityShards        = cmdFix.Flag.Int("ecParityShards", 0, "EC parity shard count for -ecx (0 = read from .vif, infer from shard count, otherwise default 4)")
	fixEcUnsafeIgnoreSidecar = cmdFix.Flag.Bool("ecUnsafeIgnoreSidecar", false, "for -ecx: proceed even when the EC bitrot checksum sidecar (.ecsum) is malformed or stale, instead of failing closed; the reconstructed shards are not verified against it")
)

type VolumeFileScanner4Fix struct {
	version        needle.Version
	nm             *needle_map.MemDb
	nmDeleted      *needle_map.MemDb
	includeDeleted bool
}

func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock super_block.SuperBlock) error {
	scanner.version = superBlock.Version
	return nil
}

func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
	return false
}

func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
	glog.V(2).Infof("key %v offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
	if n.Size.IsValid() {
		if pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size); pe != nil {
			return fmt.Errorf("saved %d with error %v", n.Size, pe)
		}
	} else {
		if scanner.includeDeleted {
			if pe := scanner.nmDeleted.Set(n.Id, types.ToOffset(offset), types.TombstoneFileSize); pe != nil {
				return fmt.Errorf("saved deleted %d with error %v", n.Size, pe)
			}
		} else {
			glog.V(2).Infof("skipping deleted file ...")
			return scanner.nm.Delete(n.Id)
		}
	}
	return nil
}

func runFix(cmd *Command, args []string) bool {
	for _, arg := range args {
		basePath, f := path.Split(util.ResolvePath(arg))
		if util.FolderExists(arg) {
			basePath = arg
			f = ""
		}

		files := []fs.DirEntry{}
		if f == "" {
			fileInfo, err := os.ReadDir(basePath)
			if err != nil {
				fmt.Println(err)
				return false
			}
			files = fileInfo
		} else {
			fileInfo, err := os.Stat(arg)
			if err != nil {
				fmt.Println(err)
				return false
			}
			files = []fs.DirEntry{fs.FileInfoToDirEntry(fileInfo)}
		}

		ext := ".dat"
		if *fixRemoteFile {
			ext = ".idx"
			util.LoadConfiguration("master", false)
			backend.LoadConfiguration(util.GetViper())
		}

		for _, file := range files {
			if !strings.HasSuffix(file.Name(), ext) {
				continue
			}
			if *fixVolumeCollection != "" {
				if !strings.HasPrefix(file.Name(), *fixVolumeCollection+"_") {
					continue
				}
			}
			baseFileName := file.Name()[:len(file.Name())-4]
			collection, volumeIdStr := "", baseFileName
			if sepIndex := strings.LastIndex(baseFileName, "_"); sepIndex > 0 {
				collection = baseFileName[:sepIndex]
				volumeIdStr = baseFileName[sepIndex+1:]
			}
			volumeId, parseErr := strconv.ParseInt(volumeIdStr, 10, 64)
			if parseErr != nil {
				fmt.Printf("Failed to parse volume id from %s: %v\n", baseFileName, parseErr)
				return false
			}
			if *fixVolumeId != 0 && *fixVolumeId != volumeId {
				continue
			}
			doFixOneVolume(basePath, baseFileName, collection, volumeId, *fixIncludeDeleted)
		}

		if *fixGenerateEcx {
			if !fixEcxFromShardsInDir(basePath, files) {
				return false
			}
		}
	}
	return true
}

// fixEcxFromShardsInDir finds EC volumes in files (identified by their .ec00
// data shard) and regenerates the .ecx (and .vif when missing) for each,
// honoring the -collection and -volumeId filters.
func fixEcxFromShardsInDir(basePath string, files []fs.DirEntry) bool {
	const shard0Ext = ".ec00"
	for _, file := range files {
		if !strings.HasSuffix(file.Name(), shard0Ext) {
			continue
		}
		if *fixVolumeCollection != "" {
			if !strings.HasPrefix(file.Name(), *fixVolumeCollection+"_") {
				continue
			}
		}
		baseFileName := file.Name()[:len(file.Name())-len(shard0Ext)]
		collection, volumeIdStr := "", baseFileName
		if sepIndex := strings.LastIndex(baseFileName, "_"); sepIndex > 0 {
			collection = baseFileName[:sepIndex]
			volumeIdStr = baseFileName[sepIndex+1:]
		}
		volumeId, parseErr := strconv.ParseInt(volumeIdStr, 10, 64)
		if parseErr != nil {
			fmt.Printf("Failed to parse volume id from %s: %v\n", baseFileName, parseErr)
			return false
		}
		if *fixVolumeId != 0 && *fixVolumeId != volumeId {
			continue
		}
		doFixEcxFromShards(basePath, baseFileName, collection, volumeId)
	}
	return true
}

func SaveToIdx(scaner *VolumeFileScanner4Fix, idxName string) (ret error) {
	idxFile, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer func() {
		idxFile.Close()
	}()

	// Emit entries in .dat offset (append) order so the .idx stays the
	// append-ordered log the volume server writes at runtime — not sorted by
	// key, which is the .sdx / .ecx shape. A key-sorted .idx puts the
	// highest-key needle last instead of the .dat-tail needle, which used to
	// flip volumes read-only on load. Every tombstone is included (including
	// any whose live needle is gone) so the last entry is the real .dat tail.
	var values []needle_map.NeedleValue
	collect := func(value needle_map.NeedleValue) error {
		values = append(values, value)
		return nil
	}
	if err = scaner.nm.AscendingVisit(collect); err != nil {
		return err
	}
	if scaner.includeDeleted {
		if err = scaner.nmDeleted.AscendingVisit(collect); err != nil {
			return err
		}
	}
	sort.Slice(values, func(i, j int) bool {
		return values[i].Offset.ToActualOffset() < values[j].Offset.ToActualOffset()
	})
	for _, value := range values {
		if _, err = idxFile.Write(value.ToBytes()); err != nil {
			return err
		}
	}
	return nil
}

func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64, fixIncludeDeleted bool) {
	indexFileName := path.Join(basepath, baseFileName+".idx")

	nm := needle_map.NewMemDb()
	nmDeleted := needle_map.NewMemDb()
	defer nm.Close()
	defer nmDeleted.Close()

	// Validate volumeId range before converting to uint32
	if volumeId < 0 || volumeId > 0xFFFFFFFF {
		err := fmt.Errorf("volume ID out of range: %d", volumeId)
		if *fixIgnoreError {
			glog.Error(err)
			return
		} else {
			glog.Fatal(err)
		}
	}
	// lgtm[go/incorrect-integer-conversion]
	// Safe conversion: volumeId has been validated to be in range [0, 0xFFFFFFFF] above
	vid := needle.VolumeId(volumeId)
	scanner := &VolumeFileScanner4Fix{
		nm:             nm,
		nmDeleted:      nmDeleted,
		includeDeleted: fixIncludeDeleted,
	}

	if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
		err := fmt.Errorf("scan .dat File: %w", err)
		if *fixIgnoreError {
			glog.Error(err)
		} else {
			glog.Fatal(err)
		}
	}

	if err := SaveToIdx(scanner, indexFileName); err != nil {
		err := fmt.Errorf("save to .idx File: %w", err)
		if *fixIgnoreError {
			glog.Error(err)
		} else {
			if err := os.Remove(indexFileName); err != nil {
				glog.Errorf("failed to cleanup file %s:%v", indexFileName, err)
			}
			glog.Fatal(err)
		}
	}
}

// doFixEcxFromShards rebuilds the sealed EC index (.ecx) for one EC volume
// directly from its local shards when both the .ecx and the original .dat are
// gone but the shards survive. When some data shards are missing but at least
// dataShards shards survive in total, the missing shards are first reconstructed
// from the survivors via Reed-Solomon. It then de-stripes the data shards into a
// temporary .dat, scans the needles, and writes a fresh ascending-sorted .ecx
// that matches what WriteSortedFileFromIdx emits at encode time (live entries
// only). When the .vif is also missing it is regenerated from the inferred EC
// ratio and the .dat size discovered during the scan.
func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int64) {
	base := path.Join(basePath, baseFileName)

	fail := func(err error) {
		if *fixIgnoreError {
			glog.Error(err)
		} else {
			glog.Fatal(err)
		}
	}

	ecxName := base + ".ecx"
	if info, err := os.Stat(ecxName); err == nil && info.Size() > 0 {
		glog.Infof("volume %d: %s already exists (%d bytes), skipping; remove it first to force regeneration", volumeId, ecxName, info.Size())
		return
	}

	// Discover which shards are present and their common size. Reed-Solomon
	// requires every shard to be the same size.
	present := make([]bool, erasure_coding.MaxShardCount)
	presentCount := 0
	maxPresentIdx := -1
	var shardSize int64
	for i := 0; i < erasure_coding.MaxShardCount; i++ {
		info, statErr := os.Stat(base + erasure_coding.ToExt(i))
		if statErr != nil || info.Size() == 0 {
			continue
		}
		if shardSize == 0 {
			shardSize = info.Size()
		} else if info.Size() != shardSize {
			fail(fmt.Errorf("volume %d: shard %s size %d does not match %d", volumeId, base+erasure_coding.ToExt(i), info.Size(), shardSize))
			return
		}
		present[i] = true
		presentCount++
		maxPresentIdx = i
	}
	if presentCount == 0 {
		fail(fmt.Errorf("volume %d: no EC shards found under %s", volumeId, base))
		return
	}

	// Resolve the EC ratio and the original .dat size.
	// Priority: explicit flags > existing .vif > defaults (10+4).
	vifName := base + ".vif"
	vifExists := util.FileExists(vifName)
	// Whether the on-disk .vif actually answers the layout question. An empty
	// stub (MaybeLoadVolumeInfo reads it as absent) or one with no EC config
	// exists but tells us nothing, and the recovered layout must still be
	// written back over it.
	vifUsable := false
	dataShards := erasure_coding.DataShardsCount
	parityShards := erasure_coding.ParityShardsCount
	var datFileSize int64
	blockSize := int64(-1) // the shard block layout; 0 legacy, >0 uniform, <0 unknown
	if vifExists {
		// MaybeLoadVolumeInfo returns a non-nil error when the .vif exists but
		// cannot be read or unmarshalled; fail loudly rather than silently
		// falling back to defaults (which would be wrong for a custom ratio).
		if vi, _, found, loadErr := volume_info.MaybeLoadVolumeInfo(vifName); loadErr != nil {
			fail(fmt.Errorf("volume %d: read %s: %w", volumeId, vifName, loadErr))
			return
		} else if found && vi != nil {
			// A partial config (parity 0, or counts past the shard ceiling)
			// describes no layout: leave the sentinel unknown and let the
			// sidecar / dual scan below answer, and rewrite the .vif at the end
			// rather than trusting it.
			cfg := vi.GetEcShardConfig()
			if cfg != nil && erasure_coding.ValidEcShardCounts(cfg.GetDataShards(), cfg.GetParityShards()) {
				dataShards = int(cfg.GetDataShards())
				parityShards = int(cfg.GetParityShards())
				// Only an EC config answers the layout question. Reading 0 off a
				// vif with no config would assert "legacy" and suppress both the
				// .ecsum fallback and the dual-layout scan below; leave the
				// sentinel at -1 (unknown) instead.
				// A recorded block size no encoder could have produced is not
				// an answer: a positive one would pin the scan to a geometry
				// that de-stripes to garbage, and a negative one would leave
				// the invalid .vif in place after the dual scan recovers the
				// real layout. Leave the sentinel unknown and rewrite the file
				// at the end.
				if bs := cfg.GetBlockSize(); erasure_coding.ValidateBlockSize(bs) == nil {
					blockSize = bs
					vifUsable = true
				} else {
					glog.Warningf("volume %d: %s records an invalid shard block size %d; recovering the layout by scan and rewriting it",
						volumeId, vifName, bs)
				}
			}
			datFileSize = vi.GetDatFileSize()
		}
	}
	// The bitrot sidecar records the same EC config at encode time; use it when
	// the .vif is gone. A uniform-layout volume cannot be de-striped correctly
	// without its block size.
	if blockSize < 0 {
		sidecarPath := erasure_coding.BitrotSidecarPath(base, 0)
		if prot, serr := erasure_coding.LoadBitrotSidecar(sidecarPath); serr == nil {
			// The sidecar is about to pin the reconstruction to one geometry
			// instead of letting the dual scan decide, so every field it
			// contributes has to hold up: it must describe generation 0 (the
			// shards being read), a complete in-range ratio, and a block size
			// an encoder could have produced. Anything less leaves the layout
			// unknown, which is the answer that still recovers by scanning.
			cfg := prot.GetEcShardConfig()
			ds, ps := int(cfg.GetDataShards()), int(cfg.GetParityShards())
			switch {
			case prot.GetGeneration() != 0:
				glog.Warningf("volume %d: %s records generation %d, not the generation-0 shards; ignoring it",
					volumeId, sidecarPath, prot.GetGeneration())
			case !erasure_coding.ValidEcShardCounts(cfg.GetDataShards(), cfg.GetParityShards()):
				glog.Warningf("volume %d: %s records invalid shard counts %d+%d; ignoring it",
					volumeId, sidecarPath, ds, ps)
			case erasure_coding.ValidateBlockSize(cfg.GetBlockSize()) != nil:
				glog.Warningf("volume %d: %s records an invalid shard block size %d; ignoring it",
					volumeId, sidecarPath, cfg.GetBlockSize())
			default:
				dataShards = ds
				parityShards = ps
				blockSize = cfg.GetBlockSize()
			}
		}
	}
	if *fixEcDataShards > 0 {
		dataShards = *fixEcDataShards
	}
	if *fixEcParityShards > 0 {
		parityShards = *fixEcParityShards
	}
	// Ensure the configured total covers every shard index actually present
	// (a custom-ratio volume with more than the default 14 shards and no .vif).
	// This never lowers parity below the default, so the common 10+4 case stays
	// correct for any subset of missing shards.
	if maxPresentIdx+1 > dataShards+parityShards {
		parityShards = maxPresentIdx + 1 - dataShards
	}
	if dataShards <= 0 || parityShards <= 0 || dataShards+parityShards > erasure_coding.MaxShardCount {
		fail(fmt.Errorf("volume %d: cannot determine EC ratio (data=%d parity=%d); set -ecDataShards/-ecParityShards", volumeId, dataShards, parityShards))
		return
	}

	// Need at least dataShards shards (any data+parity mix) to recover anything.
	if presentCount < dataShards {
		fail(fmt.Errorf("volume %d: only %d shards present, need at least %d (data shards) to recover", volumeId, presentCount, dataShards))
		return
	}

	// If any data shard is missing, reconstruct the missing shards from the
	// survivors via Reed-Solomon before de-striping. This writes the rebuilt
	// shard files back to disk, fully repairing the volume locally.
	dataComplete := true
	for i := 0; i < dataShards; i++ {
		if !present[i] {
			dataComplete = false
			break
		}
	}
	if !dataComplete {
		ctx := &erasure_coding.ECContext{DataShards: dataShards, ParityShards: parityShards}
		if blockSize > 0 {
			ctx.BlockSize = blockSize
		}
		glog.Infof("volume %d: %d/%d shards present; reconstructing missing shards (%s) before index rebuild", volumeId, presentCount, dataShards+parityShards, ctx.String())
		if _, err := erasure_coding.RebuildEcFiles(base, ctx, *fixEcUnsafeIgnoreSidecar); err != nil {
			fail(fmt.Errorf("volume %d: reconstruct missing shards from %d survivors: %w", volumeId, presentCount, err))
			return
		}
	}

	// Collect the data shards (now all present).
	shardFileNames := make([]string, dataShards)
	for i := 0; i < dataShards; i++ {
		shardPath := base + erasure_coding.ToExt(i)
		if !util.FileExists(shardPath) {
			fail(fmt.Errorf("volume %d: data shard %s still missing after reconstruction", volumeId, shardPath))
			return
		}
		shardFileNames[i] = shardPath
	}

	// Without a recorded original size, reconstruct the fully padded layout; the
	// scan below detects the trailing zero padding and recovers the true size.
	reconstructSize := datFileSize
	if reconstructSize <= 0 {
		reconstructSize = int64(dataShards) * shardSize
		glog.V(0).Infof("volume %d: no .dat size in .vif; reconstructing padded .dat (%d bytes) from %d data shards", volumeId, reconstructSize, dataShards)
	}

	// De-stripe the data shards into a temporary .dat next to the shards and
	// scan it into a fresh .ecx. With the layout unknown, de-stripe under both
	// candidate layouts and keep the one whose needle chain scans furthest.
	type layoutCandidate struct {
		name         string
		large, small int64
	}
	var candidates []layoutCandidate
	switch {
	case blockSize > 0:
		candidates = []layoutCandidate{{"uniform", blockSize, blockSize}}
	case blockSize == 0:
		candidates = []layoutCandidate{{"legacy", erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize}}
	default:
		candidates = []layoutCandidate{
			{"legacy", erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize},
		}
		// A uniform-layout shard is exactly one block long, and every block
		// size an encoder can produce is a whole number of small blocks. An
		// extent that is not — a truncated or partially copied shard — could
		// not have come from a uniform encode, and recording it would write a
		// .vif that ValidateBlockSize refuses on the next mount: the volume
		// this tool was run to rescue would never open again.
		if erasure_coding.ValidateBlockSize(shardSize) == nil && shardSize > 0 {
			candidates = append(candidates, layoutCandidate{"uniform", shardSize, shardSize})
			glog.Infof("volume %d: no .vif or .ecsum records the shard block layout; trying both", volumeId)
		} else {
			glog.Infof("volume %d: no .vif or .ecsum records the shard block layout, and the %d-byte shard extent is not a whole number of %d-byte blocks; trying the legacy layout only",
				volumeId, shardSize, erasure_coding.ErasureCodingSmallBlockSize)
		}
	}

	tmpBase := base + ".ecxrecover"
	tmpDat := tmpBase + ".dat"
	defer os.Remove(tmpDat)
	bestIdx := -1
	var realDatSize, bestNeedles int64
	var version needle.Version
	var candErr error
	for i, cand := range candidates {
		if err := erasure_coding.WriteDatFile(tmpBase, reconstructSize, reconstructSize, shardFileNames, cand.large, cand.small); err != nil {
			candErr = fmt.Errorf("volume %d: reconstruct .dat (%s layout): %w", volumeId, cand.name, err)
			continue
		}
		size, needles, ver, err := writeEcxFromDat(tmpDat, ecxName+"."+cand.name)
		if err != nil {
			os.Remove(ecxName + "." + cand.name)
			candErr = fmt.Errorf("volume %d: build .ecx from reconstructed .dat (%s layout): %w", volumeId, cand.name, err)
			continue
		}
		// The wrong layout de-stripes to garbage past the first block
		// boundary, so the layout that indexes more valid needles wins.
		// On a tie — garbage bytes do sometimes parse as plausible sizes — the
		// layout whose needle chain reached further into the .dat wins, which is
		// the distance the scan actually validated.
		if bestIdx < 0 || needles > bestNeedles || (needles == bestNeedles && size > realDatSize) {
			bestIdx = i
			bestNeedles = needles
			realDatSize = size
			version = ver
		}
	}
	if bestIdx < 0 {
		fail(candErr)
		return
	}
	for i := range candidates {
		if i != bestIdx {
			os.Remove(ecxName + "." + candidates[i].name)
		}
	}
	if err := os.Rename(ecxName+"."+candidates[bestIdx].name, ecxName); err != nil {
		fail(fmt.Errorf("volume %d: publish %s: %w", volumeId, ecxName, err))
		return
	}
	if blockSize < 0 {
		if candidates[bestIdx].name == "uniform" {
			blockSize = shardSize
		} else {
			blockSize = 0
		}
		glog.Infof("volume %d: %s layout indexes %d needles over %d bytes; keeping it", volumeId, candidates[bestIdx].name, bestNeedles, realDatSize)
	}
	glog.Infof("volume %d: wrote %s from %d data shards", volumeId, ecxName, dataShards)

	// Regenerate the .vif when it is missing OR unusable (an empty stub, or one
	// with no EC config): the layout just recovered by the dual scan is the only
	// record of it, and leaving the stub in place would mount the volume as
	// legacy on the next start and serve the wrong offsets.
	if !vifUsable {
		size := datFileSize
		if size <= 0 {
			size = realDatSize
		}
		// Never publish a layout the mount path will refuse: NewEcVolume runs
		// the same check and fails closed, so an unvalidated write here would
		// trade a recoverable volume for one that can no longer be opened.
		if bsErr := erasure_coding.ValidateBlockSize(blockSize); bsErr != nil {
			fail(fmt.Errorf("volume %d: refusing to write %s: %w", volumeId, vifName, bsErr))
			return
		}
		volumeInfo := &volume_server_pb.VolumeInfo{
			Version:     uint32(version),
			DatFileSize: size,
			EcShardConfig: &volume_server_pb.EcShardConfig{
				DataShards:   uint32(dataShards),
				ParityShards: uint32(parityShards),
				BlockSize:    blockSize,
			},
		}
		if err := volume_info.SaveVolumeInfo(vifName, volumeInfo); err != nil {
			fail(fmt.Errorf("volume %d: write %s: %w", volumeId, vifName, err))
			return
		}
		glog.Infof("volume %d: wrote %s (version %d, datFileSize %d, ec %d+%d)", volumeId, vifName, version, size, dataShards, parityShards)
	}
}

// writeEcxFromDat scans a (reconstructed) .dat and writes an ascending-sorted
// .ecx containing only live needles — the same on-disk shape
// WriteSortedFileFromIdx produces when an EC volume is first encoded. It returns
// the physical .dat size (the offset where the EC zero padding begins), the
// number of live needles indexed, and the volume version read from the
// superblock.
func writeEcxFromDat(datPath, ecxPath string) (datFileSize int64, liveNeedles int64, version needle.Version, err error) {
	f, err := os.OpenFile(datPath, os.O_RDONLY, 0644)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("open %s: %w", datPath, err)
	}
	datBackend := backend.NewDiskFile(f)
	defer datBackend.Close()

	superBlock, err := super_block.ReadSuperBlock(datBackend)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("read superblock: %w", err)
	}
	version = superBlock.Version

	fileSize, _, err := datBackend.GetStat()
	if err != nil {
		return 0, 0, version, fmt.Errorf("stat %s: %w", datPath, err)
	}

	nm := needle_map.NewMemDb()
	defer nm.Close()

	offset := int64(superBlock.BlockSize())
	for offset < fileSize {
		n, _, rest, readErr := needle.ReadNeedleHeader(datBackend, version, offset)
		if readErr != nil {
			if readErr == io.EOF {
				break
			}
			return 0, 0, version, fmt.Errorf("read needle header at offset %d: %w", offset, readErr)
		}
		// EC encoding zero-pads the tail of the last block row. An all-zero
		// header marks the start of that padding, i.e. the end of real needles.
		if n.Cookie == 0 && n.Id == 0 && n.Size == 0 {
			break
		}
		if n.Size.IsValid() {
			if pe := nm.Set(n.Id, types.ToOffset(offset), n.Size); pe != nil {
				return 0, 0, version, fmt.Errorf("set needle %d: %w", n.Id, pe)
			}
		} else {
			// Deleted/invalid: drop it so the .ecx carries only live entries,
			// matching the encode-time WriteSortedFileFromIdx behavior.
			if pe := nm.Delete(n.Id); pe != nil {
				return 0, 0, version, fmt.Errorf("delete needle %d: %w", n.Id, pe)
			}
		}
		offset += types.NeedleHeaderSize + rest
	}
	datFileSize = offset

	ecxFile, err := os.OpenFile(ecxPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return 0, 0, version, fmt.Errorf("open %s: %w", ecxPath, err)
	}
	defer ecxFile.Close()

	if err := nm.AscendingVisit(func(value needle_map.NeedleValue) error {
		liveNeedles++
		_, writeErr := ecxFile.Write(value.ToBytes())
		return writeErr
	}); err != nil {
		return 0, 0, version, fmt.Errorf("write %s: %w", ecxPath, err)
	}

	return datFileSize, liveNeedles, version, nil
}