mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-17 12:46:59 +00:00
Chris LuandGitHub
c2591b4395
* volume: verify before destroy in VolumeCopy and replication repair Four data-safety fixes around copy/repair paths that could destroy or resurrect data before verifying the source or survivors. (a) VolumeCopy no longer deletes a pre-existing local replica up front. The delete is deferred until ReadVolumeFileStatus on the source succeeds, so a transient source outage (or a retry after one) can no longer wipe a healthy destination replica. Gated on source readability only; size/count comparisons are intentionally not used because they invert legitimately after divergent vacuum/compaction. Mirrored in the Rust volume server. (b) volume.check.disk no longer resurrects vacuumed-deleted needles. A key present-and-live on the source but entirely absent on the target is ambiguous: it may be a genuine missing write, or a needle deleted on the target and then vacuumed (its index entry and any tombstone are gone). An individual needle AppendAtNs has no monotonic relation to a vacuum watermark, so the old cutoff heuristic could not tell them apart. Without positive proof the absence is a missing write, the safe default is to NOT push it back. Tradeoff: a real missing write may go unrepaired until a tombstone-aware path exists, but we never raise back deleted data. (c) Over-replication trim no longer resurrects needles or removes the wrong replica. The pre-delete sync now runs read-only (divergence check only) instead of writing the doomed replica's needles into the survivor. pickOneReplicaToDelete only ever removes the smallest of multiple healthy writable replicas; it refuses the trim when doing so would leave only read-only/integrity-flagged survivors, since file_count>0 alone cannot prove the survivor's .dat is readable. (d) Incomplete-volume (.note) cleanup keeps the shared .vif when an .ecx for the same vid coexists on the disk, so removing an interrupted regular copy cannot strip a coexisting EC volume's info file. VolumeCopy now surfaces .note write/remove errors instead of ignoring them. In the Rust volume server (where a persisting note is actually reachable) the .note check moves below the empty-stub sweep and EC validation, keeps the .vif on EC coexistence, and the mount path fails when a .note still persists. * shell: scope the over-replication writable-survivor guard to the trim path only The writable-survivor guard (never trim down to a read-only survivor) lived inside the shared pickOneReplicaToDelete, so it also gated the misplaced-volume relocation via pickOneMisplacedVolume -- a misplaced read-only volume (e.g. a full one) would silently stop being rebalanced. Extract pickSmallestReplica for the relocation path (which deletes-and-recreates and must act on read-only replicas), and keep the writable-survivor guard only in pickOneReplicaToDelete used by the over-replication trim. * seaweed-volume: recompute keep_vif after invalid-EC cleanup in the .note path keep_vif used the pre-validation ecx_exists snapshot, so when the EC-validation step above removed the invalid .ecx/shards, the .note cleanup still preserved a now-orphaned .vif. Re-check .ecx existence at cleanup time, matching the Go hasEcxFile re-check. * shell: keep placement when picking an over-replication victim to delete The trim picked the smallest writable replica without regard to placement, so it could delete the only replica in a required failure domain (e.g. with "100" and replicas dc1 + two in dc2, deleting dc1 leaves both survivors in dc2). Prefer a writable replica whose removal still satisfies placement, falling back to the smallest writable only when none does.
76 lines
2.9 KiB
Go
76 lines
2.9 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
"google.golang.org/grpc/metadata"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
// fakeVolumeCopyStream is a no-op VolumeServer_VolumeCopyServer; VolumeCopy
|
|
// errors out before sending anything in this test.
|
|
type fakeVolumeCopyStream struct {
|
|
grpc.ServerStream
|
|
}
|
|
|
|
func (s *fakeVolumeCopyStream) Send(*volume_server_pb.VolumeCopyResponse) error { return nil }
|
|
func (s *fakeVolumeCopyStream) Context() context.Context { return context.Background() }
|
|
func (s *fakeVolumeCopyStream) SetHeader(metadata.MD) error { return nil }
|
|
func (s *fakeVolumeCopyStream) SendHeader(metadata.MD) error { return nil }
|
|
func (s *fakeVolumeCopyStream) SetTrailer(metadata.MD) {}
|
|
func (s *fakeVolumeCopyStream) SendMsg(any) error { return nil }
|
|
func (s *fakeVolumeCopyStream) RecvMsg(any) error { return nil }
|
|
|
|
// TestVolumeCopy_KeepsExistingReplicaWhenSourceUnreachable verifies the
|
|
// verify-before-destroy invariant: a pre-existing healthy local replica must
|
|
// NOT be deleted when the source cannot be reached. The pre-fix code deleted
|
|
// the destination up front (and, on retry, could lose the volume entirely);
|
|
// the fix defers the delete until the source ReadVolumeFileStatus succeeds.
|
|
func TestVolumeCopy_KeepsExistingReplicaWhenSourceUnreachable(t *testing.T) {
|
|
dir := t.TempDir()
|
|
store := storage.NewStore(
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
"127.0.0.1", 0, 0, "", "test-store",
|
|
[]string{dir}, []int32{10}, []util.MinFreeSpace{{}},
|
|
dir, storage.NeedleMapInMemory,
|
|
[]types.DiskType{types.HardDriveType}, [][]string{nil},
|
|
0, stats.DiskIOProbeConfig{},
|
|
)
|
|
|
|
const vid = needle.VolumeId(42)
|
|
if err := store.AddVolume(vid, "", storage.NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0); err != nil {
|
|
t.Fatalf("AddVolume: %v", err)
|
|
}
|
|
if store.GetVolume(vid) == nil {
|
|
t.Fatalf("setup: volume %d should exist", vid)
|
|
}
|
|
|
|
vs := &VolumeServer{
|
|
store: store,
|
|
grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
}
|
|
|
|
// 127.0.0.1:1 is unreachable, so ReadVolumeFileStatus on the source fails.
|
|
req := &volume_server_pb.VolumeCopyRequest{
|
|
VolumeId: uint32(vid),
|
|
SourceDataNode: "127.0.0.1:1",
|
|
}
|
|
err := vs.VolumeCopy(req, &fakeVolumeCopyStream{})
|
|
if err == nil {
|
|
t.Fatalf("VolumeCopy should fail when the source is unreachable")
|
|
}
|
|
|
|
if store.GetVolume(vid) == nil {
|
|
t.Fatalf("existing replica %d was destroyed before the source was verified", vid)
|
|
}
|
|
}
|