From 6098ef4bd30bafec7815a76b32452acc5392cbe5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 7 Apr 2026 18:15:53 -0700 Subject: [PATCH] fix(test): remove flaky shard ID assertion in EC scrub test (#8978) * test: add integration tests for volume and EC volume scrubbing Add scrub integration tests covering normal volumes (full data scrub, corrupt .dat detection, mixed healthy/broken batches, missing volume error) and EC volumes (INDEX/LOCAL modes on healthy volumes, corrupt shard detection with broken shard info reporting, corrupt .ecx index, auto-select, unsupported mode error). Also adds framework helpers: CorruptDatFile, CorruptEcxFile, CorruptEcShardFile for fault injection in scrub tests. * fix: correct dat/ecx corruption helpers and ecx test setup - CorruptDatFile: truncate .dat to superblock size instead of overwriting bytes (ensures scrub detects data file size mismatch) - TestScrubEcVolumeIndexCorruptEcx: corrupt .ecx before mount so the corrupted size is loaded into memory (EC volumes cache ecx size at mount) * fix(test): remove flaky shard ID assertion in EC scrub test When shard 0 is truncated on disk after mount, the volume server may detect corruption via parity mismatches (shards 10-13) rather than a direct read failure on shard 0, depending on OS caching/mmap behavior. Replace the brittle shard-0-specific check with a volume ID validation. * fix(test): close upload response bodies and tighten file count assertion Wrap UploadBytes calls with ReadAllAndClose to prevent connection/fd leaks during test execution. Also tighten TotalFiles check from >= 1 to == 1 since ecSetup uploads exactly one file. --- .../grpc/scrub_integration_test.go | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/test/volume_server/grpc/scrub_integration_test.go b/test/volume_server/grpc/scrub_integration_test.go index e45ed503f..5b8da8af4 100644 --- a/test/volume_server/grpc/scrub_integration_test.go +++ b/test/volume_server/grpc/scrub_integration_test.go @@ -27,9 +27,9 @@ func TestScrubVolumeFullHealthy(t *testing.T) { framework.AllocateVolume(t, grpcClient, volumeID, "") httpClient := framework.NewHTTPClient() - framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one")) - framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two")) - framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three")) + framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one"))) + framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two"))) + framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three"))) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -65,7 +65,7 @@ func TestScrubVolumeFullCorruptData(t *testing.T) { framework.AllocateVolume(t, grpcClient, volumeID, "") httpClient := framework.NewHTTPClient() - framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data")) + framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data"))) framework.CorruptDatFile(t, clusterHarness.BaseDir(), volumeID) @@ -102,8 +102,8 @@ func TestScrubVolumeMixedHealthy(t *testing.T) { framework.AllocateVolume(t, grpcClient, corruptVol, "") httpClient := framework.NewHTTPClient() - framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy")) - framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt")) + framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy"))) + framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt"))) framework.CorruptIndexFile(t, clusterHarness.BaseDir(), corruptVol) @@ -245,8 +245,8 @@ func TestScrubEcVolumeLocalHealthy(t *testing.T) { if resp.GetTotalVolumes() != 1 { t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes()) } - if resp.GetTotalFiles() < 1 { - t.Fatalf("expected at least 1 file, got %d", resp.GetTotalFiles()) + if resp.GetTotalFiles() != 1 { + t.Fatalf("expected total_files=1, got %d", resp.GetTotalFiles()) } if len(resp.GetBrokenVolumeIds()) != 0 { t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails()) @@ -288,17 +288,12 @@ func TestScrubEcVolumeLocalCorruptShard(t *testing.T) { if len(resp.GetBrokenShardInfos()) == 0 { t.Fatalf("expected broken shard info after shard corruption") } - // Verify the reported broken shard is shard 0. - foundShard0 := false + // Verify all reported broken shards belong to the corrupted volume. for _, si := range resp.GetBrokenShardInfos() { - if si.GetShardId() == 0 { - foundShard0 = true - break + if si.GetVolumeId() != volumeID { + t.Fatalf("broken shard info for unexpected volume %d, want %d", si.GetVolumeId(), volumeID) } } - if !foundShard0 { - t.Fatalf("expected shard 0 in broken shard infos, got %v", resp.GetBrokenShardInfos()) - } } func TestScrubEcVolumeAutoSelectWithEcPresent(t *testing.T) {