mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-18 21:26:56 +00:00
* fix(ec): never delete recoverable shards on startup/reconcile (size-direction + byte-exact .dat)
EC startup validation and the cross-disk reconcile could delete the only
copy of distributed-EC shards whenever a non-empty .dat sat beside them.
This is the same data-loss class as the empty-.dat-stub fix, now for a
real (non-empty) stale or partial .dat.
validateEcVolume: the discriminating signal is the shard size relative to
the .dat's full encode, not the shard count.
- shards smaller than expected: an interrupted local encode left partial
shards and the .dat is the complete source -> reclaim the .dat.
- shards equal to expected: a valid (or still-distributing) EC volume ->
keep; the shards may be the only copy.
- shards larger than expected: the .dat is the stale/partial side (e.g. an
interrupted decode left a half-written .dat next to the real shards) ->
keep.
Previously any size mismatch, a low shard count beside a .dat, or a
transient stat error returned "delete", wiping sole-copy shards. Now every
ambiguity (size mismatch in either direction, inconsistent shard sizes,
transient I/O error, partial shard set) keeps the data; only a credible
full source .dat with no partial set to lose is reclaimed.
handleFoundEcxFile: a shard load failure (corrupt/locked .ecx, EMFILE
during a mass restart, transient I/O) no longer deletes the EC files when a
.dat exists -- it only unloads and keeps the files for retry. All deletion
authority now flows through validateEcVolume.
pruneIncompleteEcWithSiblingDat: count shards NODE-WIDE (a set split across
sibling disks summing to >= dataShards is independently recoverable and is
left alone), and require the sibling .dat to byte-exactly match the size
.vif recorded at encode time before deleting -- the prior "at least this
big, or bigger than a superblock" gate could trust a stale .dat and wipe
sole-copy shards. EC encode records the source size in .vif, so this gate
works for real volumes; older volumes without it fail safe (kept).
Rust volume server mirrors all of the above: size-direction + keep-on-
ambiguity in validate_ec_volume, keep-on-load-failure in
handle_found_ecx_file, and the node-wide + byte-exact gate in the prune.
The Rust validate/prune paths now resolve the data-shard count from the
volume's own .vif instead of hardcoding 10+4, so custom-ratio volumes are
not mis-sized and wrongly deleted on reboot.
Existing tests that encoded the old (unsafe) "delete on low count / size
mismatch" behavior are updated to the safe expectation, and new regression
tests cover the partial-decode-.dat-keeps-shards and transient-error-keeps
cases (Go and Rust); they fail on the pre-fix code.
* fix(ec): record DatFileSize in planted EC .vif for the prune test; trim comments
The multi-disk lifecycle e2e test planted a partial EC leftover with an
empty .vif, so the byte-exact prune gate (which a real encoded volume
satisfies via its recorded source size) kept it instead of cleaning up.
Record DatFileSize + the EC ratio in the planted .vif, matching production.
Also condense the verbose comments added in this change to the repo's
concise style.
750 lines
24 KiB
Go
750 lines
24 KiB
Go
package storage
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
// closeEcVolumes closes all EC volumes in the given DiskLocation to release file handles.
|
|
func closeEcVolumes(dl *DiskLocation) {
|
|
for _, ecVol := range dl.ecVolumes {
|
|
ecVol.Close()
|
|
}
|
|
}
|
|
|
|
// TestIncompleteEcEncodingCleanup tests the cleanup logic for incomplete EC encoding scenarios
|
|
func TestIncompleteEcEncodingCleanup(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
volumeId needle.VolumeId
|
|
collection string
|
|
createDatFile bool
|
|
createEcxFile bool
|
|
createEcjFile bool
|
|
numShards int
|
|
expectCleanup bool
|
|
expectLoadSuccess bool
|
|
}{
|
|
{
|
|
name: "Incomplete EC: shards without .ecx, .dat exists - should cleanup",
|
|
volumeId: 100,
|
|
collection: "",
|
|
createDatFile: true,
|
|
createEcxFile: false,
|
|
createEcjFile: false,
|
|
numShards: 14, // All shards but no .ecx
|
|
expectCleanup: true,
|
|
expectLoadSuccess: false,
|
|
},
|
|
{
|
|
name: "Distributed EC: shards without .ecx, .dat deleted - should NOT cleanup",
|
|
volumeId: 101,
|
|
collection: "",
|
|
createDatFile: false,
|
|
createEcxFile: false,
|
|
createEcjFile: false,
|
|
numShards: 5, // Partial shards, distributed
|
|
expectCleanup: false,
|
|
expectLoadSuccess: false,
|
|
},
|
|
{
|
|
// Full-size shards beside a .dat are NOT an interrupted local
|
|
// encode (which leaves equally-truncated shards smaller than the
|
|
// .dat); they may be sole copies of a distributed volume, so the
|
|
// safe behavior is to keep them rather than delete on a low count.
|
|
name: "Distributed EC: full-size shards with .ecx, < 10 of them, .dat exists - keep",
|
|
volumeId: 102,
|
|
collection: "",
|
|
createDatFile: true,
|
|
createEcxFile: true,
|
|
createEcjFile: false,
|
|
numShards: 7, // Less than DataShardsCount (10), but full size
|
|
expectCleanup: false,
|
|
expectLoadSuccess: false,
|
|
},
|
|
{
|
|
name: "Valid local EC: shards with .ecx, >= 10 shards, .dat exists - should load",
|
|
volumeId: 103,
|
|
collection: "",
|
|
createDatFile: true,
|
|
createEcxFile: true,
|
|
createEcjFile: false,
|
|
numShards: 14, // All shards
|
|
expectCleanup: false,
|
|
expectLoadSuccess: true, // Would succeed if .ecx was valid
|
|
},
|
|
{
|
|
name: "Distributed EC: shards with .ecx, .dat deleted - should load",
|
|
volumeId: 104,
|
|
collection: "",
|
|
createDatFile: false,
|
|
createEcxFile: true,
|
|
createEcjFile: false,
|
|
numShards: 10, // Enough shards
|
|
expectCleanup: false,
|
|
expectLoadSuccess: true, // Would succeed if .ecx was valid
|
|
},
|
|
{
|
|
name: "Incomplete EC with collection: shards without .ecx, .dat exists - should cleanup",
|
|
volumeId: 105,
|
|
collection: "test_collection",
|
|
createDatFile: true,
|
|
createEcxFile: false,
|
|
createEcjFile: false,
|
|
numShards: 14,
|
|
expectCleanup: true,
|
|
expectLoadSuccess: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
// Use per-subtest temp directory for stronger isolation
|
|
tempDir := t.TempDir()
|
|
|
|
// Create DiskLocation
|
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
|
|
diskLocation := &DiskLocation{
|
|
Directory: tempDir,
|
|
DirectoryUuid: "test-uuid",
|
|
IdxDirectory: tempDir,
|
|
DiskType: types.HddType,
|
|
MaxVolumeCount: 100,
|
|
OriginalMaxVolumeCount: 100,
|
|
MinFreeSpace: minFreeSpace,
|
|
}
|
|
diskLocation.volumes = make(map[needle.VolumeId]*Volume)
|
|
diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
|
|
|
|
// Setup test files
|
|
baseFileName := erasure_coding.EcShardFileName(tt.collection, tempDir, int(tt.volumeId))
|
|
|
|
// Use deterministic but small size: 10MB .dat => 1MB per shard
|
|
datFileSize := int64(10 * 1024 * 1024) // 10MB
|
|
expectedShardSize := calculateExpectedShardSize(datFileSize, erasure_coding.DataShardsCount)
|
|
|
|
// Create .dat file if needed
|
|
if tt.createDatFile {
|
|
datFile, err := os.Create(baseFileName + ".dat")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .dat file: %v", err)
|
|
}
|
|
if err := datFile.Truncate(datFileSize); err != nil {
|
|
t.Fatalf("Failed to truncate .dat file: %v", err)
|
|
}
|
|
if err := datFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close .dat file: %v", err)
|
|
}
|
|
}
|
|
|
|
// Create EC shard files
|
|
for i := 0; i < tt.numShards; i++ {
|
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
|
|
if err != nil {
|
|
t.Fatalf("Failed to create shard file: %v", err)
|
|
}
|
|
if err := shardFile.Truncate(expectedShardSize); err != nil {
|
|
t.Fatalf("Failed to truncate shard file: %v", err)
|
|
}
|
|
if err := shardFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close shard file: %v", err)
|
|
}
|
|
}
|
|
|
|
// Create .ecx file if needed
|
|
if tt.createEcxFile {
|
|
ecxFile, err := os.Create(baseFileName + ".ecx")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .ecx file: %v", err)
|
|
}
|
|
if _, err := ecxFile.WriteString("dummy ecx data"); err != nil {
|
|
ecxFile.Close()
|
|
t.Fatalf("Failed to write .ecx file: %v", err)
|
|
}
|
|
if err := ecxFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close .ecx file: %v", err)
|
|
}
|
|
}
|
|
|
|
// Create .ecj file if needed
|
|
if tt.createEcjFile {
|
|
ecjFile, err := os.Create(baseFileName + ".ecj")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .ecj file: %v", err)
|
|
}
|
|
if _, err := ecjFile.WriteString("dummy ecj data"); err != nil {
|
|
ecjFile.Close()
|
|
t.Fatalf("Failed to write .ecj file: %v", err)
|
|
}
|
|
if err := ecjFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close .ecj file: %v", err)
|
|
}
|
|
}
|
|
|
|
// Run loadAllEcShards
|
|
loadErr := diskLocation.loadAllEcShards(nil)
|
|
if loadErr != nil {
|
|
t.Logf("loadAllEcShards returned error (expected in some cases): %v", loadErr)
|
|
}
|
|
|
|
// Close EC volumes before idempotency test to avoid leaking file handles
|
|
closeEcVolumes(diskLocation)
|
|
diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
|
|
|
|
// Test idempotency - running again should not cause issues
|
|
loadErr2 := diskLocation.loadAllEcShards(nil)
|
|
if loadErr2 != nil {
|
|
t.Logf("Second loadAllEcShards returned error: %v", loadErr2)
|
|
}
|
|
t.Cleanup(func() {
|
|
closeEcVolumes(diskLocation)
|
|
})
|
|
|
|
// Verify cleanup expectations
|
|
if tt.expectCleanup {
|
|
// Check that files were cleaned up
|
|
if util.FileExists(baseFileName + ".ecx") {
|
|
t.Errorf("Expected .ecx to be cleaned up but it still exists")
|
|
}
|
|
if util.FileExists(baseFileName + ".ecj") {
|
|
t.Errorf("Expected .ecj to be cleaned up but it still exists")
|
|
}
|
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
|
|
shardFile := baseFileName + erasure_coding.ToExt(i)
|
|
if util.FileExists(shardFile) {
|
|
t.Errorf("Expected shard %d to be cleaned up but it still exists", i)
|
|
}
|
|
}
|
|
// .dat file should still exist (not cleaned up)
|
|
if tt.createDatFile && !util.FileExists(baseFileName+".dat") {
|
|
t.Errorf("Expected .dat file to remain but it was deleted")
|
|
}
|
|
} else {
|
|
// Check that files were NOT cleaned up
|
|
for i := 0; i < tt.numShards; i++ {
|
|
shardFile := baseFileName + erasure_coding.ToExt(i)
|
|
if !util.FileExists(shardFile) {
|
|
t.Errorf("Expected shard %d to remain but it was cleaned up", i)
|
|
}
|
|
}
|
|
if tt.createEcxFile && !util.FileExists(baseFileName+".ecx") {
|
|
t.Errorf("Expected .ecx to remain but it was cleaned up")
|
|
}
|
|
}
|
|
|
|
// Verify load expectations
|
|
if tt.expectLoadSuccess {
|
|
if diskLocation.EcShardCount() == 0 {
|
|
t.Errorf("Expected EC shards to be loaded for volume %d", tt.volumeId)
|
|
}
|
|
}
|
|
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestValidateEcVolume tests the validateEcVolume function
|
|
func TestValidateEcVolume(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
|
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
|
|
diskLocation := &DiskLocation{
|
|
Directory: tempDir,
|
|
DirectoryUuid: "test-uuid",
|
|
IdxDirectory: tempDir,
|
|
DiskType: types.HddType,
|
|
MinFreeSpace: minFreeSpace,
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
volumeId needle.VolumeId
|
|
collection string
|
|
createDatFile bool
|
|
numShards int
|
|
expectValid bool
|
|
}{
|
|
{
|
|
name: "Valid: .dat exists with 10+ shards",
|
|
volumeId: 200,
|
|
collection: "",
|
|
createDatFile: true,
|
|
numShards: 10,
|
|
expectValid: true,
|
|
},
|
|
{
|
|
// Full-size shards smaller in count than dataShards may be sole
|
|
// copies of a distributed volume (a real interrupted local encode
|
|
// leaves equally-truncated shards, not full-size ones), so they
|
|
// are kept rather than deleted in favor of the .dat.
|
|
name: "Keep: .dat exists with < 10 full-size shards (possible distributed sole copies)",
|
|
volumeId: 201,
|
|
collection: "",
|
|
createDatFile: true,
|
|
numShards: 9,
|
|
expectValid: true,
|
|
},
|
|
{
|
|
name: "Valid: .dat deleted (distributed EC) with any shards",
|
|
volumeId: 202,
|
|
collection: "",
|
|
createDatFile: false,
|
|
numShards: 5,
|
|
expectValid: true,
|
|
},
|
|
{
|
|
name: "Valid: .dat deleted (distributed EC) with no shards",
|
|
volumeId: 203,
|
|
collection: "",
|
|
createDatFile: false,
|
|
numShards: 0,
|
|
expectValid: true,
|
|
},
|
|
{
|
|
name: "Invalid: zero-byte shard files should not count",
|
|
volumeId: 204,
|
|
collection: "",
|
|
createDatFile: true,
|
|
numShards: 0, // Will create 10 zero-byte files below
|
|
expectValid: false,
|
|
},
|
|
{
|
|
// Inconsistent shard sizes signal corruption or mixed generations,
|
|
// not a clean interrupted encode; deleting them could destroy the
|
|
// only copy, so validation keeps them.
|
|
name: "Keep: .dat exists with different size shards (inconsistent, not trusted for deletion)",
|
|
volumeId: 205,
|
|
collection: "",
|
|
createDatFile: true,
|
|
numShards: 10, // Will create shards with varying sizes
|
|
expectValid: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
baseFileName := erasure_coding.EcShardFileName(tt.collection, tempDir, int(tt.volumeId))
|
|
|
|
// For proper testing, we need to use realistic sizes that match EC encoding
|
|
// EC uses large blocks (1GB) and small blocks (1MB)
|
|
// For test purposes, use a small .dat file size that still exercises the logic
|
|
// 10MB .dat file = 1MB per shard (one small batch, fast and deterministic)
|
|
datFileSize := int64(10 * 1024 * 1024) // 10MB
|
|
expectedShardSize := calculateExpectedShardSize(datFileSize, erasure_coding.DataShardsCount)
|
|
|
|
// Create .dat file if needed
|
|
if tt.createDatFile {
|
|
datFile, err := os.Create(baseFileName + ".dat")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .dat file: %v", err)
|
|
}
|
|
// Write minimal data (don't need to fill entire 10GB for tests)
|
|
datFile.Truncate(datFileSize)
|
|
datFile.Close()
|
|
}
|
|
|
|
// Create EC shard files with correct size
|
|
for i := 0; i < tt.numShards; i++ {
|
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
|
|
if err != nil {
|
|
t.Fatalf("Failed to create shard file: %v", err)
|
|
}
|
|
// Use truncate to create file of correct size without allocating all the space
|
|
if err := shardFile.Truncate(expectedShardSize); err != nil {
|
|
shardFile.Close()
|
|
t.Fatalf("Failed to truncate shard file: %v", err)
|
|
}
|
|
if err := shardFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close shard file: %v", err)
|
|
}
|
|
}
|
|
|
|
// For zero-byte test case, create empty files for all data shards
|
|
if tt.volumeId == 204 {
|
|
for i := 0; i < erasure_coding.DataShardsCount; i++ {
|
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
|
|
if err != nil {
|
|
t.Fatalf("Failed to create empty shard file: %v", err)
|
|
}
|
|
// Don't write anything - leave as zero-byte
|
|
shardFile.Close()
|
|
}
|
|
}
|
|
|
|
// For mismatched shard size test case, create shards with different sizes
|
|
if tt.volumeId == 205 {
|
|
for i := 0; i < erasure_coding.DataShardsCount; i++ {
|
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
|
|
if err != nil {
|
|
t.Fatalf("Failed to create shard file: %v", err)
|
|
}
|
|
// Write different amount of data to each shard
|
|
data := make([]byte, 100+i*10)
|
|
shardFile.Write(data)
|
|
shardFile.Close()
|
|
}
|
|
}
|
|
|
|
// Test validation
|
|
isValid := diskLocation.validateEcVolume(tt.collection, tt.volumeId)
|
|
if isValid != tt.expectValid {
|
|
t.Errorf("Expected validation result %v but got %v", tt.expectValid, isValid)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestRemoveEcVolumeFiles tests the removeEcVolumeFiles function
|
|
func TestRemoveEcVolumeFiles(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
separateIdxDir bool
|
|
}{
|
|
{"Same directory for data and index", false},
|
|
{"Separate idx directory", true},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
|
|
var dataDir, idxDir string
|
|
if tt.separateIdxDir {
|
|
dataDir = filepath.Join(tempDir, "data")
|
|
idxDir = filepath.Join(tempDir, "idx")
|
|
os.MkdirAll(dataDir, 0755)
|
|
os.MkdirAll(idxDir, 0755)
|
|
} else {
|
|
dataDir = tempDir
|
|
idxDir = tempDir
|
|
}
|
|
|
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
|
|
diskLocation := &DiskLocation{
|
|
Directory: dataDir,
|
|
DirectoryUuid: "test-uuid",
|
|
IdxDirectory: idxDir,
|
|
DiskType: types.HddType,
|
|
MinFreeSpace: minFreeSpace,
|
|
}
|
|
|
|
volumeId := needle.VolumeId(300)
|
|
collection := ""
|
|
dataBaseFileName := erasure_coding.EcShardFileName(collection, dataDir, int(volumeId))
|
|
idxBaseFileName := erasure_coding.EcShardFileName(collection, idxDir, int(volumeId))
|
|
|
|
// Create all EC shard files in data directory
|
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
|
|
shardFile, err := os.Create(dataBaseFileName + erasure_coding.ToExt(i))
|
|
if err != nil {
|
|
t.Fatalf("Failed to create shard file: %v", err)
|
|
}
|
|
if _, err := shardFile.WriteString("dummy shard data"); err != nil {
|
|
shardFile.Close()
|
|
t.Fatalf("Failed to write shard file: %v", err)
|
|
}
|
|
if err := shardFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close shard file: %v", err)
|
|
}
|
|
}
|
|
|
|
// Create .ecx file in idx directory
|
|
ecxFile, err := os.Create(idxBaseFileName + ".ecx")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .ecx file: %v", err)
|
|
}
|
|
if _, err := ecxFile.WriteString("dummy ecx data"); err != nil {
|
|
ecxFile.Close()
|
|
t.Fatalf("Failed to write .ecx file: %v", err)
|
|
}
|
|
if err := ecxFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close .ecx file: %v", err)
|
|
}
|
|
|
|
// Create .ecj file in idx directory
|
|
ecjFile, err := os.Create(idxBaseFileName + ".ecj")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .ecj file: %v", err)
|
|
}
|
|
if _, err := ecjFile.WriteString("dummy ecj data"); err != nil {
|
|
ecjFile.Close()
|
|
t.Fatalf("Failed to write .ecj file: %v", err)
|
|
}
|
|
if err := ecjFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close .ecj file: %v", err)
|
|
}
|
|
|
|
// Create .dat file in data directory (should NOT be removed)
|
|
datFile, err := os.Create(dataBaseFileName + ".dat")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .dat file: %v", err)
|
|
}
|
|
if _, err := datFile.WriteString("dummy dat data"); err != nil {
|
|
datFile.Close()
|
|
t.Fatalf("Failed to write .dat file: %v", err)
|
|
}
|
|
if err := datFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close .dat file: %v", err)
|
|
}
|
|
|
|
// Call removeEcVolumeFiles
|
|
diskLocation.removeEcVolumeFiles(collection, volumeId)
|
|
|
|
// Verify all EC shard files are removed from data directory
|
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
|
|
shardFile := dataBaseFileName + erasure_coding.ToExt(i)
|
|
if util.FileExists(shardFile) {
|
|
t.Errorf("Shard file %d should be removed but still exists", i)
|
|
}
|
|
}
|
|
|
|
// Verify .ecx file is removed from idx directory
|
|
if util.FileExists(idxBaseFileName + ".ecx") {
|
|
t.Errorf(".ecx file should be removed but still exists")
|
|
}
|
|
|
|
// Verify .ecj file is removed from idx directory
|
|
if util.FileExists(idxBaseFileName + ".ecj") {
|
|
t.Errorf(".ecj file should be removed but still exists")
|
|
}
|
|
|
|
// Verify .dat file is NOT removed from data directory
|
|
if !util.FileExists(dataBaseFileName + ".dat") {
|
|
t.Errorf(".dat file should NOT be removed but was deleted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestEcCleanupWithSeparateIdxDirectory tests EC cleanup when idx directory is different
|
|
func TestEcCleanupWithSeparateIdxDirectory(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
|
|
idxDir := filepath.Join(tempDir, "idx")
|
|
dataDir := filepath.Join(tempDir, "data")
|
|
os.MkdirAll(idxDir, 0755)
|
|
os.MkdirAll(dataDir, 0755)
|
|
|
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
|
|
diskLocation := &DiskLocation{
|
|
Directory: dataDir,
|
|
DirectoryUuid: "test-uuid",
|
|
IdxDirectory: idxDir,
|
|
DiskType: types.HddType,
|
|
MinFreeSpace: minFreeSpace,
|
|
}
|
|
diskLocation.volumes = make(map[needle.VolumeId]*Volume)
|
|
diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
|
|
|
|
volumeId := needle.VolumeId(400)
|
|
collection := ""
|
|
|
|
// Create shards in data directory (shards only go to Directory, not IdxDirectory)
|
|
dataBaseFileName := erasure_coding.EcShardFileName(collection, dataDir, int(volumeId))
|
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
|
|
shardFile, err := os.Create(dataBaseFileName + erasure_coding.ToExt(i))
|
|
if err != nil {
|
|
t.Fatalf("Failed to create shard file: %v", err)
|
|
}
|
|
if _, err := shardFile.WriteString("dummy shard data"); err != nil {
|
|
t.Fatalf("Failed to write shard file: %v", err)
|
|
}
|
|
if err := shardFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close shard file: %v", err)
|
|
}
|
|
}
|
|
|
|
// Create .dat in data directory
|
|
datFile, err := os.Create(dataBaseFileName + ".dat")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .dat file: %v", err)
|
|
}
|
|
if _, err := datFile.WriteString("dummy data"); err != nil {
|
|
t.Fatalf("Failed to write .dat file: %v", err)
|
|
}
|
|
if err := datFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close .dat file: %v", err)
|
|
}
|
|
|
|
// Do not create .ecx: trigger orphaned-shards cleanup when .dat exists
|
|
|
|
// Run loadAllEcShards
|
|
loadErr := diskLocation.loadAllEcShards(nil)
|
|
if loadErr != nil {
|
|
t.Logf("loadAllEcShards error: %v", loadErr)
|
|
}
|
|
t.Cleanup(func() {
|
|
closeEcVolumes(diskLocation)
|
|
})
|
|
|
|
// Verify cleanup occurred in data directory (shards)
|
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
|
|
shardFile := dataBaseFileName + erasure_coding.ToExt(i)
|
|
if util.FileExists(shardFile) {
|
|
t.Errorf("Shard file %d should be cleaned up but still exists", i)
|
|
}
|
|
}
|
|
|
|
// Verify .dat in data directory still exists (only EC files are cleaned up)
|
|
if !util.FileExists(dataBaseFileName + ".dat") {
|
|
t.Errorf(".dat file should remain but was deleted")
|
|
}
|
|
}
|
|
|
|
// TestDistributedEcVolumeNoFileDeletion verifies that distributed EC volumes
|
|
// (where .dat is deleted) do NOT have their shard files deleted when load fails
|
|
// This tests the critical bug fix where DestroyEcVolume was incorrectly deleting files
|
|
func TestDistributedEcVolumeNoFileDeletion(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
|
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
|
|
diskLocation := &DiskLocation{
|
|
Directory: tempDir,
|
|
DirectoryUuid: "test-uuid",
|
|
IdxDirectory: tempDir,
|
|
DiskType: types.HddType,
|
|
MinFreeSpace: minFreeSpace,
|
|
ecVolumes: make(map[needle.VolumeId]*erasure_coding.EcVolume),
|
|
}
|
|
|
|
collection := ""
|
|
volumeId := needle.VolumeId(500)
|
|
baseFileName := erasure_coding.EcShardFileName(collection, tempDir, int(volumeId))
|
|
|
|
// Create EC shards (only 5 shards - less than DataShardsCount, but OK for distributed EC)
|
|
numDistributedShards := 5
|
|
for i := 0; i < numDistributedShards; i++ {
|
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i))
|
|
if err != nil {
|
|
t.Fatalf("Failed to create shard file: %v", err)
|
|
}
|
|
if _, err := shardFile.WriteString("dummy shard data"); err != nil {
|
|
shardFile.Close()
|
|
t.Fatalf("Failed to write shard file: %v", err)
|
|
}
|
|
if err := shardFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close shard file: %v", err)
|
|
}
|
|
}
|
|
|
|
// Create .ecx file to trigger EC loading
|
|
ecxFile, err := os.Create(baseFileName + ".ecx")
|
|
if err != nil {
|
|
t.Fatalf("Failed to create .ecx file: %v", err)
|
|
}
|
|
if _, err := ecxFile.WriteString("dummy ecx data"); err != nil {
|
|
ecxFile.Close()
|
|
t.Fatalf("Failed to write .ecx file: %v", err)
|
|
}
|
|
if err := ecxFile.Close(); err != nil {
|
|
t.Fatalf("Failed to close .ecx file: %v", err)
|
|
}
|
|
|
|
// NO .dat file - this is a distributed EC volume
|
|
|
|
// Run loadAllEcShards - this should fail but NOT delete shard files
|
|
loadErr := diskLocation.loadAllEcShards(nil)
|
|
if loadErr != nil {
|
|
t.Logf("loadAllEcShards returned error (expected): %v", loadErr)
|
|
}
|
|
t.Cleanup(func() {
|
|
closeEcVolumes(diskLocation)
|
|
})
|
|
|
|
// CRITICAL CHECK: Verify shard files still exist (should NOT be deleted)
|
|
for i := 0; i < 5; i++ {
|
|
shardFile := baseFileName + erasure_coding.ToExt(i)
|
|
if !util.FileExists(shardFile) {
|
|
t.Errorf("CRITICAL BUG: Shard file %s was deleted for distributed EC volume!", shardFile)
|
|
}
|
|
}
|
|
|
|
// Verify .ecx file still exists (should NOT be deleted for distributed EC)
|
|
if !util.FileExists(baseFileName + ".ecx") {
|
|
t.Errorf("CRITICAL BUG: .ecx file was deleted for distributed EC volume!")
|
|
}
|
|
|
|
t.Logf("SUCCESS: Distributed EC volume files preserved (not deleted)")
|
|
}
|
|
|
|
// TestLoadExistingVolumeSkipsVifWhenEcxPresent pins the skip behavior on
|
|
// the LoadVolume / MountVolume path (skipIfEcVolumesExists=false) for the
|
|
// .vif + .ecx disk layout without .dat. Two variants cover both
|
|
// IdxDirectory==Directory and the split-idx-dir fallback.
|
|
func TestLoadExistingVolumeSkipsVifWhenEcxPresent(t *testing.T) {
|
|
const vid needle.VolumeId = 42
|
|
|
|
cases := []struct {
|
|
name string
|
|
splitDirs bool
|
|
}{
|
|
{name: "same-idx-dir", splitDirs: false},
|
|
{name: "split-idx-dir", splitDirs: true},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
dataDir := t.TempDir()
|
|
idxDir := dataDir
|
|
if tc.splitDirs {
|
|
idxDir = t.TempDir()
|
|
}
|
|
|
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
|
|
diskLocation := &DiskLocation{
|
|
Directory: dataDir,
|
|
DirectoryUuid: "test-uuid",
|
|
IdxDirectory: idxDir,
|
|
DiskType: types.HddType,
|
|
MaxVolumeCount: 100,
|
|
OriginalMaxVolumeCount: 100,
|
|
MinFreeSpace: minFreeSpace,
|
|
}
|
|
diskLocation.volumes = make(map[needle.VolumeId]*Volume)
|
|
diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
|
|
|
|
vifPath := erasure_coding.EcShardFileName("", dataDir, int(vid)) + ".vif"
|
|
ecxPath := erasure_coding.EcShardFileName("", idxDir, int(vid)) + ".ecx"
|
|
if err := os.WriteFile(vifPath, []byte{}, 0644); err != nil {
|
|
t.Fatalf("write .vif: %v", err)
|
|
}
|
|
if err := os.WriteFile(ecxPath, []byte{}, 0644); err != nil {
|
|
t.Fatalf("write .ecx: %v", err)
|
|
}
|
|
|
|
entries, err := os.ReadDir(dataDir)
|
|
if err != nil {
|
|
t.Fatalf("read dir: %v", err)
|
|
}
|
|
var vifEntry os.DirEntry
|
|
for _, e := range entries {
|
|
if filepath.Ext(e.Name()) == ".vif" {
|
|
vifEntry = e
|
|
break
|
|
}
|
|
}
|
|
if vifEntry == nil {
|
|
t.Fatalf(".vif entry missing from dir listing")
|
|
}
|
|
|
|
loaded := diskLocation.loadExistingVolume(vifEntry, NeedleMapInMemory, false, 0, 0)
|
|
if loaded {
|
|
t.Fatalf("loadExistingVolume should refuse to load a .vif-only entry when .ecx is present (volume %d)", vid)
|
|
}
|
|
if _, exists := diskLocation.volumes[vid]; exists {
|
|
t.Fatalf("volume %d should not be registered in l.volumes (would create phantom regular volume)", vid)
|
|
}
|
|
datPath := erasure_coding.EcShardFileName("", dataDir, int(vid)) + ".dat"
|
|
if util.FileExists(datPath) {
|
|
t.Fatalf("guard must not create a placeholder .dat for volume %d", vid)
|
|
}
|
|
})
|
|
}
|
|
}
|