mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-05-24 02:31:28 +00:00
43dbebfa04
Move removed-replica drain and replica-scoped invalidation onto explicit core-command paths so the widened multi-replica runtime no longer depends on coarse host-side recovery handling. Made-with: Cursor
466 lines
13 KiB
Go
466 lines
13 KiB
Go
package blockcmd
|
|
|
|
import (
|
|
"errors"
|
|
"reflect"
|
|
"testing"
|
|
|
|
engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
|
|
)
|
|
|
|
type fakeOps struct {
|
|
applyRoleFn func(blockvol.BlockVolumeAssignment) (bool, error)
|
|
startReceiverFn func(blockvol.BlockVolumeAssignment) (bool, error)
|
|
configureShipperFn func(string, []engine.ReplicaAssignment) (bool, bool, error)
|
|
startRecoveryTaskFn func(string, blockvol.BlockVolumeAssignment) (bool, error)
|
|
drainRecoveryTaskFn func(string, string) (bool, error)
|
|
invalidateSessionFn func(string, string, string) (bool, error)
|
|
startCatchUpFn func(string, uint64) (bool, error)
|
|
startRebuildFn func(string, uint64) (bool, error)
|
|
}
|
|
|
|
func (f fakeOps) ApplyRole(a blockvol.BlockVolumeAssignment) (bool, error) {
|
|
if f.applyRoleFn == nil {
|
|
return false, nil
|
|
}
|
|
return f.applyRoleFn(a)
|
|
}
|
|
|
|
func (f fakeOps) StartReceiver(a blockvol.BlockVolumeAssignment) (bool, error) {
|
|
if f.startReceiverFn == nil {
|
|
return false, nil
|
|
}
|
|
return f.startReceiverFn(a)
|
|
}
|
|
|
|
func (f fakeOps) ConfigureShipper(volumeID string, replicas []engine.ReplicaAssignment) (bool, bool, error) {
|
|
if f.configureShipperFn == nil {
|
|
return false, false, nil
|
|
}
|
|
return f.configureShipperFn(volumeID, replicas)
|
|
}
|
|
|
|
func (f fakeOps) StartRecoveryTask(replicaID string, assignment blockvol.BlockVolumeAssignment) (bool, error) {
|
|
if f.startRecoveryTaskFn == nil {
|
|
return false, nil
|
|
}
|
|
return f.startRecoveryTaskFn(replicaID, assignment)
|
|
}
|
|
|
|
func (f fakeOps) DrainRecoveryTask(replicaID, reason string) (bool, error) {
|
|
if f.drainRecoveryTaskFn == nil {
|
|
return false, nil
|
|
}
|
|
return f.drainRecoveryTaskFn(replicaID, reason)
|
|
}
|
|
|
|
func (f fakeOps) InvalidateSession(volumeID, replicaID, reason string) (bool, error) {
|
|
if f.invalidateSessionFn == nil {
|
|
return false, nil
|
|
}
|
|
return f.invalidateSessionFn(volumeID, replicaID, reason)
|
|
}
|
|
|
|
func (f fakeOps) StartCatchUp(replicaID string, targetLSN uint64) (bool, error) {
|
|
if f.startCatchUpFn == nil {
|
|
return false, nil
|
|
}
|
|
return f.startCatchUpFn(replicaID, targetLSN)
|
|
}
|
|
|
|
func (f fakeOps) StartRebuild(replicaID string, targetLSN uint64) (bool, error) {
|
|
if f.startRebuildFn == nil {
|
|
return false, nil
|
|
}
|
|
return f.startRebuildFn(replicaID, targetLSN)
|
|
}
|
|
|
|
type fakeEffects struct {
|
|
recorded []string
|
|
events []engine.Event
|
|
published map[string]engine.PublicationProjection
|
|
}
|
|
|
|
func (f *fakeEffects) RecordCommand(volumeID, name string) {
|
|
f.recorded = append(f.recorded, volumeID+":"+name)
|
|
}
|
|
|
|
func (f *fakeEffects) EmitCoreEvent(ev engine.Event) {
|
|
f.events = append(f.events, ev)
|
|
}
|
|
|
|
func (f *fakeEffects) PublishProjection(volumeID string, projection engine.PublicationProjection) error {
|
|
if f.published == nil {
|
|
f.published = make(map[string]engine.PublicationProjection)
|
|
}
|
|
f.published[volumeID] = projection
|
|
return nil
|
|
}
|
|
|
|
func TestDispatcher_ApplyRoleRequiresMatchingAssignment(t *testing.T) {
|
|
d := NewDispatcher(fakeOps{}, &fakeEffects{})
|
|
err := d.Run([]engine.Command{engine.ApplyRoleCommand{VolumeID: "vol1"}}, &blockvol.BlockVolumeAssignment{Path: "other"})
|
|
if err == nil {
|
|
t.Fatal("expected path mismatch error")
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_StartReceiverRecordsAndEmitsObserved(t *testing.T) {
|
|
effects := &fakeEffects{}
|
|
d := NewDispatcher(fakeOps{
|
|
startReceiverFn: func(a blockvol.BlockVolumeAssignment) (bool, error) {
|
|
return true, nil
|
|
},
|
|
}, effects)
|
|
assignment := &blockvol.BlockVolumeAssignment{
|
|
Path: "vol1",
|
|
ReplicaDataAddr: "10.0.0.1:9333",
|
|
ReplicaCtrlAddr: "10.0.0.1:9334",
|
|
}
|
|
if err := d.Run([]engine.Command{engine.StartReceiverCommand{VolumeID: "vol1"}}, assignment); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(effects.recorded, []string{"vol1:start_receiver"}) {
|
|
t.Fatalf("recorded=%v", effects.recorded)
|
|
}
|
|
if len(effects.events) != 1 {
|
|
t.Fatalf("events=%d", len(effects.events))
|
|
}
|
|
if _, ok := effects.events[0].(engine.ReceiverReadyObserved); !ok {
|
|
t.Fatalf("event=%T", effects.events[0])
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_ConfigureShipperKeepsHostEffectsServerSide(t *testing.T) {
|
|
effects := &fakeEffects{}
|
|
d := NewDispatcher(fakeOps{
|
|
configureShipperFn: func(volumeID string, replicas []engine.ReplicaAssignment) (bool, bool, error) {
|
|
return true, true, nil
|
|
},
|
|
}, effects)
|
|
err := d.Run([]engine.Command{engine.ConfigureShipperCommand{
|
|
VolumeID: "vol1",
|
|
Replicas: []engine.ReplicaAssignment{{ReplicaID: "vol1/vs2", Endpoint: engine.Endpoint{DataAddr: "data", CtrlAddr: "ctrl"}}},
|
|
}}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(effects.recorded, []string{"vol1:configure_shipper"}) {
|
|
t.Fatalf("recorded=%v", effects.recorded)
|
|
}
|
|
if len(effects.events) != 2 {
|
|
t.Fatalf("events=%d", len(effects.events))
|
|
}
|
|
if _, ok := effects.events[0].(engine.ShipperConfiguredObserved); !ok {
|
|
t.Fatalf("event0=%T", effects.events[0])
|
|
}
|
|
if _, ok := effects.events[1].(engine.ShipperConnectedObserved); !ok {
|
|
t.Fatalf("event1=%T", effects.events[1])
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_PublishProjectionUsesHostEffect(t *testing.T) {
|
|
effects := &fakeEffects{}
|
|
d := NewDispatcher(fakeOps{}, effects)
|
|
proj := engine.PublicationProjection{VolumeID: "vol1", Role: engine.RolePrimary}
|
|
if err := d.Run([]engine.Command{engine.PublishProjectionCommand{VolumeID: "vol1", Projection: proj}}, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, ok := effects.published["vol1"]
|
|
if !ok || got.VolumeID != "vol1" || got.Role != engine.RolePrimary {
|
|
t.Fatalf("published=%v", effects.published)
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_StopsOnFirstError(t *testing.T) {
|
|
effects := &fakeEffects{}
|
|
d := NewDispatcher(fakeOps{
|
|
startCatchUpFn: func(replicaID string, targetLSN uint64) (bool, error) {
|
|
return false, errors.New("boom")
|
|
},
|
|
startRebuildFn: func(replicaID string, targetLSN uint64) (bool, error) {
|
|
t.Fatal("should not execute after first error")
|
|
return false, nil
|
|
},
|
|
}, effects)
|
|
err := d.Run([]engine.Command{
|
|
engine.StartCatchUpCommand{VolumeID: "vol1", ReplicaID: "vol1/r1", TargetLSN: 10},
|
|
engine.StartRebuildCommand{VolumeID: "vol1", ReplicaID: "vol1/r1", TargetLSN: 20},
|
|
}, nil)
|
|
if err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
}
|
|
|
|
type fakeRecoveryCoordinator struct {
|
|
startedReplica string
|
|
startedAssigns []blockvol.BlockVolumeAssignment
|
|
drained []struct {
|
|
replicaID string
|
|
reason string
|
|
}
|
|
catchUpCalls []struct {
|
|
replicaID string
|
|
targetLSN uint64
|
|
}
|
|
rebuildCalls []struct {
|
|
replicaID string
|
|
targetLSN uint64
|
|
}
|
|
}
|
|
|
|
func (f *fakeRecoveryCoordinator) StartRecoveryTask(replicaID string, assignments []blockvol.BlockVolumeAssignment) {
|
|
f.startedReplica = replicaID
|
|
f.startedAssigns = assignments
|
|
}
|
|
|
|
func (f *fakeRecoveryCoordinator) DrainRecoveryTask(replicaID, reason string) {
|
|
f.drained = append(f.drained, struct {
|
|
replicaID string
|
|
reason string
|
|
}{replicaID: replicaID, reason: reason})
|
|
}
|
|
|
|
func (f *fakeRecoveryCoordinator) ExecutePendingCatchUp(replicaID string, targetLSN uint64) error {
|
|
f.catchUpCalls = append(f.catchUpCalls, struct {
|
|
replicaID string
|
|
targetLSN uint64
|
|
}{replicaID: replicaID, targetLSN: targetLSN})
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRecoveryCoordinator) ExecutePendingRebuild(replicaID string, targetLSN uint64) error {
|
|
f.rebuildCalls = append(f.rebuildCalls, struct {
|
|
replicaID string
|
|
targetLSN uint64
|
|
}{replicaID: replicaID, targetLSN: targetLSN})
|
|
return nil
|
|
}
|
|
|
|
type fakeProjectionReader struct {
|
|
proj engine.PublicationProjection
|
|
ok bool
|
|
}
|
|
|
|
func (f fakeProjectionReader) Projection(volumeID string) (engine.PublicationProjection, bool) {
|
|
if !f.ok || f.proj.VolumeID != volumeID {
|
|
return engine.PublicationProjection{}, false
|
|
}
|
|
return f.proj, true
|
|
}
|
|
|
|
type fakeProjectionCache struct {
|
|
published map[string]engine.PublicationProjection
|
|
}
|
|
|
|
func (f *fakeProjectionCache) StoreProjection(volumeID string, projection engine.PublicationProjection) {
|
|
if f.published == nil {
|
|
f.published = make(map[string]engine.PublicationProjection)
|
|
}
|
|
f.published[volumeID] = projection
|
|
}
|
|
|
|
type fakeSessionInvalidator struct {
|
|
reasons []string
|
|
states []engine.ReplicaState
|
|
}
|
|
|
|
func (f *fakeSessionInvalidator) InvalidateSession(reason string, targetState engine.ReplicaState) {
|
|
f.reasons = append(f.reasons, reason)
|
|
f.states = append(f.states, targetState)
|
|
}
|
|
|
|
func TestServiceOps_StartRecoveryTaskUsesRecoveryCoordinator(t *testing.T) {
|
|
rec := &fakeRecoveryCoordinator{}
|
|
ops := NewServiceOps(fakeOps{}, rec, nil, nil)
|
|
assign := blockvol.BlockVolumeAssignment{Path: "vol1"}
|
|
executed, err := ops.StartRecoveryTask("vol1/vs2", assign)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !executed {
|
|
t.Fatal("expected executed")
|
|
}
|
|
if rec.startedReplica != "vol1/vs2" || len(rec.startedAssigns) != 1 || rec.startedAssigns[0].Path != "vol1" {
|
|
t.Fatalf("started=%q assigns=%v", rec.startedReplica, rec.startedAssigns)
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_DrainRecoveryTaskRecordsExecution(t *testing.T) {
|
|
effects := &fakeEffects{}
|
|
var drained []string
|
|
d := NewDispatcher(fakeOps{
|
|
drainRecoveryTaskFn: func(replicaID, reason string) (bool, error) {
|
|
drained = append(drained, replicaID+":"+reason)
|
|
return true, nil
|
|
},
|
|
}, effects)
|
|
err := d.Run([]engine.Command{
|
|
engine.DrainRecoveryTaskCommand{VolumeID: "vol1", ReplicaID: "vol1/vs2", Reason: "assignment_removed"},
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(drained, []string{"vol1/vs2:assignment_removed"}) {
|
|
t.Fatalf("drained=%v", drained)
|
|
}
|
|
if !reflect.DeepEqual(effects.recorded, []string{"vol1:drain_recovery_task"}) {
|
|
t.Fatalf("recorded=%v", effects.recorded)
|
|
}
|
|
}
|
|
|
|
func TestHostEffects_PublishProjectionPrefersLatestCoreProjection(t *testing.T) {
|
|
cache := &fakeProjectionCache{}
|
|
effects := NewHostEffects(
|
|
nil,
|
|
nil,
|
|
fakeProjectionReader{
|
|
proj: engine.PublicationProjection{VolumeID: "vol1", Role: engine.RoleReplica},
|
|
ok: true,
|
|
},
|
|
cache,
|
|
)
|
|
err := effects.PublishProjection("vol1", engine.PublicationProjection{VolumeID: "vol1", Role: engine.RolePrimary})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, ok := cache.published["vol1"]
|
|
if !ok {
|
|
t.Fatal("expected cached projection")
|
|
}
|
|
if got.Role != engine.RoleReplica {
|
|
t.Fatalf("role=%v", got.Role)
|
|
}
|
|
}
|
|
|
|
func TestHostEffects_RecordAndEmitUseCallbacks(t *testing.T) {
|
|
var recorded []string
|
|
var events []engine.Event
|
|
effects := NewHostEffects(
|
|
func(volumeID, name string) {
|
|
recorded = append(recorded, volumeID+":"+name)
|
|
},
|
|
func(ev engine.Event) {
|
|
events = append(events, ev)
|
|
},
|
|
nil,
|
|
nil,
|
|
)
|
|
effects.RecordCommand("vol1", "apply_role")
|
|
effects.EmitCoreEvent(engine.RoleApplied{ID: "vol1"})
|
|
if !reflect.DeepEqual(recorded, []string{"vol1:apply_role"}) {
|
|
t.Fatalf("recorded=%v", recorded)
|
|
}
|
|
if len(events) != 1 {
|
|
t.Fatalf("events=%d", len(events))
|
|
}
|
|
if _, ok := events[0].(engine.RoleApplied); !ok {
|
|
t.Fatalf("event=%T", events[0])
|
|
}
|
|
}
|
|
|
|
func TestServiceOps_InvalidateSessionUsesProjectionAndSenderResolver(t *testing.T) {
|
|
s1 := &fakeSessionInvalidator{}
|
|
s2 := &fakeSessionInvalidator{}
|
|
ops := NewServiceOps(
|
|
fakeOps{},
|
|
nil,
|
|
fakeProjectionReader{
|
|
ok: true,
|
|
proj: engine.PublicationProjection{
|
|
VolumeID: "vol1",
|
|
ReplicaIDs: []string{"vol1/vs2", "vol1/vs3"},
|
|
},
|
|
},
|
|
func(replicaID string) SessionInvalidator {
|
|
switch replicaID {
|
|
case "vol1/vs2":
|
|
return s1
|
|
case "vol1/vs3":
|
|
return s2
|
|
default:
|
|
return nil
|
|
}
|
|
},
|
|
)
|
|
executed, err := ops.InvalidateSession("vol1", "", "test_reason")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !executed {
|
|
t.Fatal("expected executed")
|
|
}
|
|
if !reflect.DeepEqual(s1.reasons, []string{"test_reason"}) || !reflect.DeepEqual(s2.reasons, []string{"test_reason"}) {
|
|
t.Fatalf("reasons1=%v reasons2=%v", s1.reasons, s2.reasons)
|
|
}
|
|
if len(s1.states) != 1 || s1.states[0] != engine.StateDisconnected {
|
|
t.Fatalf("states1=%v", s1.states)
|
|
}
|
|
}
|
|
|
|
func TestServiceOps_InvalidateSessionTargetsSingleReplicaWhenProvided(t *testing.T) {
|
|
s1 := &fakeSessionInvalidator{}
|
|
s2 := &fakeSessionInvalidator{}
|
|
ops := NewServiceOps(
|
|
fakeOps{},
|
|
nil,
|
|
fakeProjectionReader{
|
|
ok: true,
|
|
proj: engine.PublicationProjection{
|
|
VolumeID: "vol1",
|
|
ReplicaIDs: []string{"vol1/vs2", "vol1/vs3"},
|
|
},
|
|
},
|
|
func(replicaID string) SessionInvalidator {
|
|
switch replicaID {
|
|
case "vol1/vs2":
|
|
return s1
|
|
case "vol1/vs3":
|
|
return s2
|
|
default:
|
|
return nil
|
|
}
|
|
},
|
|
)
|
|
executed, err := ops.InvalidateSession("vol1", "vol1/vs2", "test_reason")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !executed {
|
|
t.Fatal("expected executed")
|
|
}
|
|
if !reflect.DeepEqual(s1.reasons, []string{"test_reason"}) {
|
|
t.Fatalf("reasons1=%v", s1.reasons)
|
|
}
|
|
if len(s2.reasons) != 0 {
|
|
t.Fatalf("reasons2=%v", s2.reasons)
|
|
}
|
|
}
|
|
|
|
func TestServiceOps_DrainRecoveryTaskUsesRecoveryCoordinator(t *testing.T) {
|
|
rec := &fakeRecoveryCoordinator{}
|
|
ops := NewServiceOps(fakeOps{}, rec, nil, nil)
|
|
executed, err := ops.DrainRecoveryTask("vol1/vs2", "assignment_removed")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !executed {
|
|
t.Fatal("expected executed")
|
|
}
|
|
if len(rec.drained) != 1 || rec.drained[0].replicaID != "vol1/vs2" || rec.drained[0].reason != "assignment_removed" {
|
|
t.Fatalf("drained=%v", rec.drained)
|
|
}
|
|
}
|
|
|
|
func TestServiceOps_StartRecoveryTask_NilRecoveryIsNoop(t *testing.T) {
|
|
ops := NewServiceOps(fakeOps{}, nil, nil, nil)
|
|
executed, err := ops.StartRecoveryTask("vol1/vs2", blockvol.BlockVolumeAssignment{Path: "vol1"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if executed {
|
|
t.Fatal("expected noop when recovery is nil")
|
|
}
|
|
}
|