Compare commits

...
Author SHA1 Message Date
Chris Lu d60f5201f9 refactor(ec): drop unused buffer/block-size params from generateMissingEcFiles
The function only routes through rebuildEcFiles, which uses
ErasureCodingSmallBlockSize directly. The buffer/block-size knobs
threaded in from RebuildEcFilesWithContext were never read. Drop them
from the signature and the sole caller; encoding still goes through
generateEcFiles which keeps its own copies of the same params.
2026-05-09 14:45:44 -07:00
Chris Lu 4e058c2bce refactor(ec): drop unused shardPaths slice in generateMissingEcFiles
The slice was populated for present shards but never read. Drop it;
ghost residue is already tracked separately and the input file handle
is the only thing the rebuild loop consumes.
2026-05-09 14:45:08 -07:00
Chris Lu 849d7949a3 fix(ec): normalize paths when matching ghost residue
Use filepath.Clean on both the freshly opened output file names and
the ghost paths so that lexically different but equivalent paths
(e.g. "./datadir/X.ec00" vs "datadir/X.ec00") still match. Without
this, a redundant entry in additionalDirs could let a ghost path slip
through the overwritten check and os.Remove a freshly rebuilt shard.
2026-05-09 14:44:47 -07:00
Chris Lu 1690bb2a03 test(ec): cover 8-disk multi-disk rebuild and end-to-end decode (#9340)
Mirror the reporter's topology — single volume server with 8 hdd
locations (/mnt/d{1..8}/weed), .ecx and shards spread across disks the
way the volume server places them after a few rebuild rounds — and
exercise three states: clean, 0-byte residue on the rebuild disk,
0-byte residue on a sibling disk. Each variant must rebuild the missing
shards byte-identical to the originals AND decode the data shards back
through WriteDatFile to a .dat that matches the original byte-for-byte,
which is the real safety check that non-zero rebuilt files aren't
masking corrupted content.
2026-05-08 10:00:48 -07:00
Chris Lu 59228cf809 perf(ec): pre-allocate missing-shard buffers across rebuild chunks
Reconstruct previously got nil for each missing shard and allocated a
fresh 1MB buffer per chunk — thousands of allocations on a 1GB volume.
klauspost/reedsolomon reuses cap when len==0, so handing it a 0-len
slice with the existing capacity avoids the allocation while still
signaling "missing".
2026-05-07 23:29:03 -07:00
Chris Lu 79164c8eef refactor(ec): return shard size from findShardFile to skip second stat
generateMissingEcFiles was os.Stat'ing each shard a second time just to
read the size after findShardFile already had it. Return the size
directly so the inner loop is one Stat per shard instead of two.
2026-05-07 22:25:40 -07:00
Chris Lu b122308bf4 fix(ec): skip 0-byte residue and verify shard sizes before rebuild (#9340)
A previously aborted ec.rebuild leaves 0-byte placeholder files for the
shards it was about to write. On the next attempt, findShardFile saw
those ghosts as "present", the read loop returned nil on the first n==0,
and rebuild claimed success while leaving the volume mounted with empty
stripes — manifesting downstream as "too few shards given" or silent
corruption.

Now: skip 0-byte files when scanning, validate that all surviving shards
agree on size, bound the read loop by that size instead of bailing on
the first short read, and remove residue after a successful rebuild so
it can't shadow real shards next time.
2026-05-06 16:09:35 -07:00
2 changed files with 539 additions and 49 deletions
+94 -49
View File
@@ -13,7 +13,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (
@@ -100,7 +99,7 @@ func RebuildEcFiles(baseFileName string, additionalDirs ...string) ([]uint32, er
// RebuildEcFilesWithContext rebuilds missing EC files using the provided context.
// additionalDirs are extra directories to search for existing shard files.
func RebuildEcFilesWithContext(baseFileName string, ctx *ECContext, additionalDirs ...string) ([]uint32, error) {
return generateMissingEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx, additionalDirs)
return generateMissingEcFiles(baseFileName, ctx, additionalDirs)
}
func ToExt(ecIndex int) string {
@@ -128,44 +127,64 @@ func generateEcFiles(baseFileName string, bufferSize int, largeBlockSize int64,
}
// findShardFile looks for a shard file at baseFileName+ext, then in additionalDirs.
func findShardFile(baseFileName string, ext string, additionalDirs []string) string {
primary := baseFileName + ext
if util.FileExists(primary) {
return primary
}
// Returns the first non-empty shard file path found. Any 0-byte residue files
// (typically left behind by a previously aborted rebuild) are returned in ghosts
// so the caller can remove them — otherwise they shadow real shards on the next
// rebuild attempt and the volume gets silently mounted with empty stripes.
func findShardFile(baseFileName string, ext string, additionalDirs []string) (shardPath string, shardSize int64, ghosts []string) {
candidates := make([]string, 0, 1+len(additionalDirs))
candidates = append(candidates, baseFileName+ext)
baseName := filepath.Base(baseFileName)
for _, dir := range additionalDirs {
candidate := filepath.Join(dir, baseName+ext)
if util.FileExists(candidate) {
return candidate
candidates = append(candidates, filepath.Join(dir, baseName+ext))
}
for _, c := range candidates {
fi, statErr := os.Stat(c)
if statErr != nil {
continue
}
if fi.Size() == 0 {
ghosts = append(ghosts, c)
continue
}
if shardPath == "" {
shardPath = c
shardSize = fi.Size()
}
}
return ""
return
}
func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext, additionalDirs []string) (generatedShardIds []uint32, err error) {
func generateMissingEcFiles(baseFileName string, ctx *ECContext, additionalDirs []string) (generatedShardIds []uint32, err error) {
// Pass 1: discover which shards exist and which are missing,
// opening input files but NOT creating output files yet.
shardHasData := make([]bool, ctx.Total())
shardPaths := make([]string, ctx.Total()) // non-empty for present shards
inputFiles := make([]*os.File, ctx.Total())
var ghostPaths []string
expectedShardSize := int64(-1)
presentCount := 0
for shardId := 0; shardId < ctx.Total(); shardId++ {
ext := ctx.ToExt(shardId)
shardPath := findShardFile(baseFileName, ext, additionalDirs)
if shardPath != "" {
shardHasData[shardId] = true
shardPaths[shardId] = shardPath
inputFiles[shardId], err = os.OpenFile(shardPath, os.O_RDONLY, 0)
if err != nil {
return nil, err
}
defer inputFiles[shardId].Close()
presentCount++
} else {
shardPath, shardSize, ghosts := findShardFile(baseFileName, ext, additionalDirs)
ghostPaths = append(ghostPaths, ghosts...)
if shardPath == "" {
generatedShardIds = append(generatedShardIds, uint32(shardId))
continue
}
if expectedShardSize < 0 {
expectedShardSize = shardSize
} else if shardSize != expectedShardSize {
return nil, fmt.Errorf("ec shard size mismatch: %s is %d bytes, expected %d (refusing to rebuild from inconsistent shards)",
shardPath, shardSize, expectedShardSize)
}
shardHasData[shardId] = true
inputFiles[shardId], err = os.OpenFile(shardPath, os.O_RDONLY, 0)
if err != nil {
return nil, err
}
defer inputFiles[shardId].Close()
presentCount++
}
// Pre-check: bail out before creating any output files.
@@ -174,8 +193,8 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
baseFileName, presentCount, ctx.DataShards, generatedShardIds)
}
glog.V(0).Infof("rebuilding %s: %d shards present, %d missing %v, config %s",
baseFileName, presentCount, len(generatedShardIds), generatedShardIds, ctx.String())
glog.V(0).Infof("rebuilding %s: %d shards present (size %d), %d missing %v, %d ghost residue %v, config %s",
baseFileName, presentCount, expectedShardSize, len(generatedShardIds), generatedShardIds, len(ghostPaths), ghostPaths, ctx.String())
// Pass 2: create output files for missing shards now that we know
// reconstruction is possible.
@@ -192,10 +211,32 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
defer outputFiles[shardId].Close()
}
err = rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx)
err = rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx, expectedShardSize)
if err != nil {
return nil, fmt.Errorf("rebuildEcFiles: %w", err)
}
// Reconstruction succeeded — remove any 0-byte residue from prior aborted
// rebuilds so they don't shadow real shards on the next pass and don't get
// mounted as empty stripes. Skip residue paths that the output-file step
// has already overwritten with real data, since those entries in
// ghostPaths refer to the just-rebuilt shard's location.
freshOutputs := make(map[string]struct{}, len(outputFiles))
for _, f := range outputFiles {
if f != nil {
freshOutputs[filepath.Clean(f.Name())] = struct{}{}
}
}
for _, ghost := range ghostPaths {
if _, overwritten := freshOutputs[filepath.Clean(ghost)]; overwritten {
continue
}
if removeErr := os.Remove(ghost); removeErr != nil && !os.IsNotExist(removeErr) {
glog.Warningf("failed to remove 0-byte ec shard residue %s: %v", ghost, removeErr)
} else {
glog.V(0).Infof("removed 0-byte ec shard residue %s", ghost)
}
}
return
}
@@ -320,39 +361,42 @@ func encodeDatFile(remainingSize int64, baseFileName string, bufferSize int, lar
return nil
}
func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*os.File, ctx *ECContext) error {
func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*os.File, ctx *ECContext, expectedShardSize int64) error {
enc, err := ctx.CreateEncoder()
if err != nil {
return fmt.Errorf("failed to create encoder: %w", err)
}
// Pre-allocate buffers for every shard, including the missing ones we
// need to reconstruct. reedsolomon.Reconstruct reuses cap when len==0
// (cap(shard) >= shardSize → shard[0:shardSize]), so handing it a 0-len
// slice with the right cap avoids a fresh allocation per chunk.
buffers := make([][]byte, ctx.Total())
for i := range buffers {
if shardHasData[i] {
buffers[i] = make([]byte, ErasureCodingSmallBlockSize)
}
buffers[i] = make([]byte, ErasureCodingSmallBlockSize)
}
var startOffset int64
var inputBufferDataSize int
for {
for startOffset := int64(0); startOffset < expectedShardSize; {
chunkSize := int64(ErasureCodingSmallBlockSize)
if remaining := expectedShardSize - startOffset; remaining < chunkSize {
chunkSize = remaining
}
// read the input data from files
for i := 0; i < ctx.Total(); i++ {
if shardHasData[i] {
n, _ := inputFiles[i].ReadAt(buffers[i], startOffset)
if n == 0 {
return nil
}
if inputBufferDataSize == 0 {
inputBufferDataSize = n
}
if inputBufferDataSize != n {
return fmt.Errorf("ec shard size expected %d actual %d", inputBufferDataSize, n)
buf := buffers[i][:chunkSize]
n, readErr := inputFiles[i].ReadAt(buf, startOffset)
if int64(n) != chunkSize {
return fmt.Errorf("short read on shard %s at offset %d: got %d, want %d (err=%v)",
inputFiles[i].Name(), startOffset, n, chunkSize, readErr)
}
buffers[i] = buf
} else {
buffers[i] = nil
// 0-len, full-cap: signals "missing" while letting Reconstruct
// reslice into our existing storage.
buffers[i] = buffers[i][:0]
}
}
@@ -365,15 +409,16 @@ func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*o
// write the data to output files
for i := 0; i < ctx.Total(); i++ {
if !shardHasData[i] {
n, _ := outputFiles[i].WriteAt(buffers[i][:inputBufferDataSize], startOffset)
if inputBufferDataSize != n {
return fmt.Errorf("fail to write to %s", outputFiles[i].Name())
n, writeErr := outputFiles[i].WriteAt(buffers[i][:chunkSize], startOffset)
if int64(n) != chunkSize {
return fmt.Errorf("short write to %s at offset %d: got %d, want %d (err=%v)",
outputFiles[i].Name(), startOffset, n, chunkSize, writeErr)
}
}
}
startOffset += int64(inputBufferDataSize)
startOffset += chunkSize
}
return nil
}
func readNeedleMap(baseFileName string) (*needle_map.MemDb, error) {
@@ -0,0 +1,445 @@
package erasure_coding
import (
"bytes"
"crypto/rand"
"os"
"path/filepath"
"sort"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
)
// TestRebuildEcFiles_BasicTwoMissing covers the simple repair path: 12 of 14
// shards present, 2 truly missing — must rebuild both and produce shards
// identical to the originals.
func TestRebuildEcFiles_BasicTwoMissing(t *testing.T) {
dir := t.TempDir()
base := filepath.Join(dir, "_109")
originalShards := encodeFixture(t, base, 50*1024*1024)
// Drop shards 1 and 8.
mustRemove(t, base+ToExt(1))
mustRemove(t, base+ToExt(8))
rebuilt, err := RebuildEcFiles(base)
if err != nil {
t.Fatalf("RebuildEcFiles: %v", err)
}
sort.Slice(rebuilt, func(i, j int) bool { return rebuilt[i] < rebuilt[j] })
if want := []uint32{1, 8}; !equalUint32(rebuilt, want) {
t.Fatalf("rebuilt = %v, want %v", rebuilt, want)
}
for _, idx := range []int{1, 8} {
got := mustReadFile(t, base+ToExt(idx))
if !bytes.Equal(got, originalShards[idx]) {
t.Fatalf("shard %d content does not match the original encoding", idx)
}
}
}
// TestRebuildEcFiles_RejectsZeroByteResidue is the regression for issue #9340.
//
// When a previous ec.rebuild attempt aborts (e.g. crash, network error,
// reedsolomon "too few shards" mid-rebuild), it leaves 0-byte placeholder
// files for the shards it was about to write. On the next attempt, the old
// findShardFile saw those 0-byte ghosts as "present", short-circuited the
// reconstruction loop on the first n==0 read, and returned nil — making
// ec.rebuild claim success while leaving the volume with empty shards.
//
// The fix: skip 0-byte files when scanning, treat the shards as still-missing,
// and remove the residue after a successful rebuild so they don't shadow real
// shards on the next pass.
func TestRebuildEcFiles_RejectsZeroByteResidue(t *testing.T) {
dir := t.TempDir()
base := filepath.Join(dir, "_109")
originalShards := encodeFixture(t, base, 50*1024*1024)
// Simulate the residue from a prior aborted rebuild: shard files for the
// truly-missing shards exist on disk but are 0 bytes.
for _, idx := range []int{1, 8} {
mustRemove(t, base+ToExt(idx))
mustWriteFile(t, base+ToExt(idx), nil)
}
rebuilt, err := RebuildEcFiles(base)
if err != nil {
t.Fatalf("RebuildEcFiles: %v", err)
}
sort.Slice(rebuilt, func(i, j int) bool { return rebuilt[i] < rebuilt[j] })
if want := []uint32{1, 8}; !equalUint32(rebuilt, want) {
t.Fatalf("rebuilt = %v, want %v (the old code reported [] and silently left 0-byte residue)", rebuilt, want)
}
for _, idx := range []int{1, 8} {
fi, err := os.Stat(base + ToExt(idx))
if err != nil {
t.Fatalf("stat shard %d: %v", idx, err)
}
if fi.Size() == 0 {
t.Fatalf("shard %d still 0 bytes after rebuild — residue was not cleared", idx)
}
got := mustReadFile(t, base+ToExt(idx))
if !bytes.Equal(got, originalShards[idx]) {
t.Fatalf("rebuilt shard %d does not match the original encoding", idx)
}
}
}
// TestRebuildEcFiles_ResidueOnAdditionalDisk reproduces the multi-disk variant
// of the residue bug. The .ecx and the real shards live on the rebuild disk;
// a sibling disk holds 0-byte placeholders for the shards that need to be
// regenerated (e.g. an earlier rebuild crashed while writing them there). The
// pre-fix findShardFile preferred the first existing path it saw — including
// the 0-byte one — so the rebuild silently succeeded with empty stripes.
func TestRebuildEcFiles_ResidueOnAdditionalDisk(t *testing.T) {
root := t.TempDir()
disk1 := filepath.Join(root, "d1")
disk2 := filepath.Join(root, "d2")
mustMkdir(t, disk1)
mustMkdir(t, disk2)
stagingBase := filepath.Join(root, "_109")
originalShards := encodeFixture(t, stagingBase, 50*1024*1024)
// Real shards (except 1 and 8) and .vif live on disk1.
disk1Base := filepath.Join(disk1, "_109")
for i := 0; i < TotalShardsCount; i++ {
if i == 1 || i == 8 {
continue
}
mustRename(t, stagingBase+ToExt(i), disk1Base+ToExt(i))
}
mustRename(t, stagingBase+".vif", disk1Base+".vif")
// disk2 carries 0-byte residue for the missing shards.
disk2Base := filepath.Join(disk2, "_109")
for _, idx := range []int{1, 8} {
mustWriteFile(t, disk2Base+ToExt(idx), nil)
}
rebuilt, err := RebuildEcFiles(disk1Base, disk2)
if err != nil {
t.Fatalf("RebuildEcFiles: %v", err)
}
sort.Slice(rebuilt, func(i, j int) bool { return rebuilt[i] < rebuilt[j] })
if want := []uint32{1, 8}; !equalUint32(rebuilt, want) {
t.Fatalf("rebuilt = %v, want %v", rebuilt, want)
}
for _, idx := range []int{1, 8} {
got := mustReadFile(t, disk1Base+ToExt(idx))
if !bytes.Equal(got, originalShards[idx]) {
t.Fatalf("rebuilt shard %d on disk1 does not match the original", idx)
}
// Residue on the sibling disk must be cleaned up so it can't shadow
// the real shard on a future scan.
if _, err := os.Stat(disk2Base + ToExt(idx)); !os.IsNotExist(err) {
t.Fatalf("shard %d residue on disk2 was not removed (err=%v)", idx, err)
}
}
}
// TestRebuildEcFiles_RejectsMismatchedShardSizes ensures we surface a clear
// error when surviving shards disagree on size, instead of feeding inconsistent
// data into reedsolomon and producing corrupted output.
func TestRebuildEcFiles_RejectsMismatchedShardSizes(t *testing.T) {
dir := t.TempDir()
base := filepath.Join(dir, "_109")
encodeFixture(t, base, 50*1024*1024)
mustRemove(t, base+ToExt(1))
// Truncate one surviving shard so its size disagrees with the others.
if err := os.Truncate(base+ToExt(2), 1024); err != nil {
t.Fatalf("truncate: %v", err)
}
if _, err := RebuildEcFiles(base); err == nil {
t.Fatalf("expected size-mismatch error, got nil")
}
}
// encodeFixture writes a .dat file of the requested size, EC-encodes it, saves
// a matching .vif, and returns a snapshot of every original shard's bytes.
func encodeFixture(t *testing.T, baseFileName string, datSize int64) [TotalShardsCount][]byte {
t.Helper()
data := make([]byte, datSize)
if _, err := rand.Read(data); err != nil {
t.Fatalf("rand: %v", err)
}
if err := os.WriteFile(baseFileName+".dat", data, 0644); err != nil {
t.Fatalf("write dat: %v", err)
}
if err := WriteEcFiles(baseFileName); err != nil {
t.Fatalf("WriteEcFiles: %v", err)
}
// A minimal .vif so RebuildEcFiles picks up the right ec config rather
// than warning and falling back to defaults.
vif := &volume_server_pb.VolumeInfo{
Version: 3,
DatFileSize: datSize,
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: DataShardsCount,
ParityShards: ParityShardsCount,
},
}
if err := volume_info.SaveVolumeInfo(baseFileName+".vif", vif); err != nil {
t.Fatalf("save vif: %v", err)
}
var shards [TotalShardsCount][]byte
for i := 0; i < TotalShardsCount; i++ {
shards[i] = mustReadFile(t, baseFileName+ToExt(i))
}
return shards
}
func mustRemove(t *testing.T, path string) {
t.Helper()
if err := os.Remove(path); err != nil {
t.Fatalf("remove %s: %v", path, err)
}
}
func mustWriteFile(t *testing.T, path string, data []byte) {
t.Helper()
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
func mustReadFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return data
}
func mustRename(t *testing.T, from, to string) {
t.Helper()
if err := os.Rename(from, to); err != nil {
t.Fatalf("rename %s -> %s: %v", from, to, err)
}
}
func mustMkdir(t *testing.T, dir string) {
t.Helper()
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatalf("mkdir %s: %v", dir, err)
}
}
func equalUint32(a, b []uint32) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// TestRebuildEcFiles_EightDiskTopology reproduces the user's reported topology
// from #9340: a single volume server with 8 hdd locations
// (/mnt/d{1..8}/weed), .ecx on one disk, the surviving local shard on another,
// freshly-copied shards on a third, shards 1 and 8 truly missing. The volume
// server's VolumeEcShardsRebuild picks the .ecx-owning disk as rebuildLocation
// and passes every other disk as additionalDirs; we mirror that contract here.
//
// The test exercises:
//
// 1. Clean state — must rebuild shards 1 and 8 byte-identical to the
// pre-removal originals.
// 2. End-to-end decode — concatenating the data shards back through
// WriteDatFile reproduces the original .dat byte-for-byte, proving the
// rebuild is semantically correct (not just non-zero).
// 3. Residue scenario — a previously aborted rebuild left 0-byte placeholders
// for shards 1 and 8 on the rebuildLocation. Pre-fix this caused
// findShardFile to return the ghost, the read loop to short-circuit on
// n==0, and the rebuild to silently no-op. Post-fix the ghosts are
// skipped, the rebuild succeeds, and the residue is removed.
func TestRebuildEcFiles_EightDiskTopology(t *testing.T) {
for _, tc := range []struct {
name string
residueOn func(disks []string) [][2]string // (disk, ext) pairs to seed as 0-byte ghosts
}{
{name: "clean", residueOn: nil},
{
name: "residue_on_rebuild_disk",
residueOn: func(disks []string) [][2]string {
return [][2]string{
{disks[1], ToExt(1)}, // d2 == rebuildLocation in this fixture
{disks[1], ToExt(8)},
}
},
},
{
name: "residue_on_sibling_disk",
residueOn: func(disks []string) [][2]string {
return [][2]string{
{disks[3], ToExt(1)}, // a totally unrelated sibling disk
{disks[6], ToExt(8)},
}
},
},
} {
t.Run(tc.name, func(t *testing.T) {
runEightDiskRebuild(t, tc.residueOn)
})
}
}
func runEightDiskRebuild(t *testing.T, residueOn func(disks []string) [][2]string) {
t.Helper()
root := t.TempDir()
disks := make([]string, 8)
for i := range disks {
disks[i] = filepath.Join(root, "d"+string(rune('1'+i)), "weed")
mustMkdir(t, disks[i])
}
// Encode in a staging dir, then distribute shards across the 8 disks the
// way the volume server would after a couple of rebuild rounds:
// .ecx + .vif + a few shards on d2 (this is rebuildLocation), the
// existing shard 2 on d1, copied shards spread across d3..d8.
stagingBase := filepath.Join(root, "staging", "_109")
mustMkdir(t, filepath.Dir(stagingBase))
const datSize = int64(50 * 1024 * 1024)
originalShards, originalDat := encodeFixtureWithDat(t, stagingBase, datSize)
rebuildLocation := disks[1] // d2 — owns .ecx
homeDisk := []int{0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // d1 holds shard 2 + most copied shards
homeDisk[3], homeDisk[4], homeDisk[5], homeDisk[6], homeDisk[7] = 2, 3, 4, 5, 6 // spread a few copied shards to d3..d7
homeDisk[10], homeDisk[11], homeDisk[12], homeDisk[13] = 7, 0, 1, 2 // and the rest across d8/d1/d2/d3
missing := map[int]struct{}{1: {}, 8: {}}
rebuildBase := filepath.Join(rebuildLocation, "collect_109")
mustRename(t, stagingBase+".vif", rebuildBase+".vif")
if _, err := os.Stat(stagingBase + ".ecx"); err == nil {
mustRename(t, stagingBase+".ecx", rebuildBase+".ecx")
}
for i := 0; i < TotalShardsCount; i++ {
if _, gone := missing[i]; gone {
mustRemove(t, stagingBase+ToExt(i))
continue
}
dst := filepath.Join(disks[homeDisk[i]], "collect_109"+ToExt(i))
mustRename(t, stagingBase+ToExt(i), dst)
}
if residueOn != nil {
for _, ent := range residueOn(disks) {
ghost := filepath.Join(ent[0], "collect_109"+ent[1])
mustWriteFile(t, ghost, nil)
}
}
// Mirror VolumeEcShardsRebuild's contract: rebuildLocation is the disk
// with .ecx; every other disk goes into additionalDirs.
var additionalDirs []string
for i, d := range disks {
if i == 1 {
continue
}
additionalDirs = append(additionalDirs, d)
}
rebuilt, err := RebuildEcFiles(rebuildBase, additionalDirs...)
if err != nil {
t.Fatalf("RebuildEcFiles: %v", err)
}
sort.Slice(rebuilt, func(i, j int) bool { return rebuilt[i] < rebuilt[j] })
if want := []uint32{1, 8}; !equalUint32(rebuilt, want) {
t.Fatalf("rebuilt = %v, want %v", rebuilt, want)
}
// Each rebuilt shard must match the original encoding bit-for-bit.
for idx := range missing {
got := mustReadFile(t, rebuildBase+ToExt(idx))
if !bytes.Equal(got, originalShards[idx]) {
t.Fatalf("rebuilt shard %d does not match the original encoding", idx)
}
}
// Any 0-byte residue we seeded on a sibling disk must be removed; residue
// at the rebuild location's output path is fine — it gets overwritten.
if residueOn != nil {
for _, ent := range residueOn(disks) {
ghost := filepath.Join(ent[0], "collect_109"+ent[1])
if ent[0] == rebuildLocation {
continue // overwritten by the output file
}
if _, statErr := os.Stat(ghost); !os.IsNotExist(statErr) {
t.Fatalf("residue %s was not removed (statErr=%v)", ghost, statErr)
}
}
}
// Decode all 10 data shards back through WriteDatFile and verify the
// reconstructed .dat matches the original byte-for-byte. This is the real
// safety check — non-zero rebuilt files aren't enough; they must contain
// the right data.
dataShardPaths := make([]string, DataShardsCount)
for i := 0; i < DataShardsCount; i++ {
if _, gone := missing[i]; gone {
dataShardPaths[i] = rebuildBase + ToExt(i)
continue
}
dataShardPaths[i] = filepath.Join(disks[homeDisk[i]], "collect_109"+ToExt(i))
}
decodedBase := filepath.Join(root, "decoded", "out")
mustMkdir(t, filepath.Dir(decodedBase))
if err := WriteDatFile(decodedBase, datSize, dataShardPaths); err != nil {
t.Fatalf("WriteDatFile: %v", err)
}
decoded := mustReadFile(t, decodedBase+".dat")
if !bytes.Equal(decoded, originalDat) {
t.Fatalf("decoded .dat does not match original (len got=%d want=%d)", len(decoded), len(originalDat))
}
}
// encodeFixtureWithDat is encodeFixture but also returns the original .dat
// bytes so the caller can verify a full encode→rebuild→decode round trip.
func encodeFixtureWithDat(t *testing.T, baseFileName string, datSize int64) (shards [TotalShardsCount][]byte, dat []byte) {
t.Helper()
dat = make([]byte, datSize)
if _, err := rand.Read(dat); err != nil {
t.Fatalf("rand: %v", err)
}
if err := os.WriteFile(baseFileName+".dat", dat, 0644); err != nil {
t.Fatalf("write dat: %v", err)
}
if err := WriteEcFiles(baseFileName); err != nil {
t.Fatalf("WriteEcFiles: %v", err)
}
if err := WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil {
// No .idx in this fixture (we don't run any reads), so this is best-effort.
t.Logf("WriteSortedFileFromIdx (best-effort): %v", err)
}
vif := &volume_server_pb.VolumeInfo{
Version: 3,
DatFileSize: datSize,
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: DataShardsCount,
ParityShards: ParityShardsCount,
},
}
if err := volume_info.SaveVolumeInfo(baseFileName+".vif", vif); err != nil {
t.Fatalf("save vif: %v", err)
}
for i := 0; i < TotalShardsCount; i++ {
shards[i] = mustReadFile(t, baseFileName+ToExt(i))
}
return
}