mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-08-19 05:36:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): remove flaky shard ID assertion in EC scrub test (#8978)

Browse Source 
* test: add integration tests for volume and EC volume scrubbing

Add scrub integration tests covering normal volumes (full data scrub,
corrupt .dat detection, mixed healthy/broken batches, missing volume
error) and EC volumes (INDEX/LOCAL modes on healthy volumes, corrupt
shard detection with broken shard info reporting, corrupt .ecx index,
auto-select, unsupported mode error).

Also adds framework helpers: CorruptDatFile, CorruptEcxFile,
CorruptEcShardFile for fault injection in scrub tests.

* fix: correct dat/ecx corruption helpers and ecx test setup

- CorruptDatFile: truncate .dat to superblock size instead of overwriting
  bytes (ensures scrub detects data file size mismatch)
- TestScrubEcVolumeIndexCorruptEcx: corrupt .ecx before mount so the
  corrupted size is loaded into memory (EC volumes cache ecx size at mount)

* fix(test): remove flaky shard ID assertion in EC scrub test

When shard 0 is truncated on disk after mount, the volume server may
detect corruption via parity mismatches (shards 10-13) rather than a
direct read failure on shard 0, depending on OS caching/mmap behavior.
Replace the brittle shard-0-specific check with a volume ID validation.

* fix(test): close upload response bodies and tighten file count assertion

Wrap UploadBytes calls with ReadAllAndClose to prevent connection/fd
leaks during test execution. Also tighten TotalFiles check from >= 1
to == 1 since ecSetup uploads exactly one file.
This commit is contained in:
Chris Lu
2026-04-07 18:15:53 -07:00
committed by GitHub
parent 4bf6d195e4
commit 6098ef4bd3
1 changed files with 11 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/volume_server/grpc/scrub_integration_test.go
+11 -16
View File
@@ -27,9 +27,9 @@ func TestScrubVolumeFullHealthy(t *testing.T) {
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one"))
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two"))
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three"))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one")))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two")))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three")))
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@@ -65,7 +65,7 @@ func TestScrubVolumeFullCorruptData(t *testing.T) {
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data"))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data")))
framework.CorruptDatFile(t, clusterHarness.BaseDir(), volumeID)
@@ -102,8 +102,8 @@ func TestScrubVolumeMixedHealthy(t *testing.T) {
framework.AllocateVolume(t, grpcClient, corruptVol, "")
httpClient := framework.NewHTTPClient()
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy"))
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt"))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy")))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt")))
framework.CorruptIndexFile(t, clusterHarness.BaseDir(), corruptVol)
@@ -245,8 +245,8 @@ func TestScrubEcVolumeLocalHealthy(t *testing.T) {
if resp.GetTotalVolumes() != 1 {
t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
}
if resp.GetTotalFiles() < 1 {
t.Fatalf("expected at least 1 file, got %d", resp.GetTotalFiles())
if resp.GetTotalFiles() != 1 {
t.Fatalf("expected total_files=1, got %d", resp.GetTotalFiles())
}
if len(resp.GetBrokenVolumeIds()) != 0 {
t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
@@ -288,17 +288,12 @@ func TestScrubEcVolumeLocalCorruptShard(t *testing.T) {
if len(resp.GetBrokenShardInfos()) == 0 {
t.Fatalf("expected broken shard info after shard corruption")
}
// Verify the reported broken shard is shard 0.
foundShard0 := false
// Verify all reported broken shards belong to the corrupted volume.
for _, si := range resp.GetBrokenShardInfos() {
if si.GetShardId() == 0 {
foundShard0 = true
break
if si.GetVolumeId() != volumeID {
t.Fatalf("broken shard info for unexpected volume %d, want %d", si.GetVolumeId(), volumeID)
}
}
if !foundShard0 {
t.Fatalf("expected shard 0 in broken shard infos, got %v", resp.GetBrokenShardInfos())
}
}
func TestScrubEcVolumeAutoSelectWithEcPresent(t *testing.T) {