mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-19 14:34:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: proactive shipper reconnect on rejoin (Bug 5)

Browse Source 
After rejoin, the shipper is configured but no I/O triggers Ship(),
so the shipper stays Disconnected and the core stays at
awaiting_shipper_connected indefinitely.

Fix: observePrimaryShipperConnectivity now calls TryReconnectShippers
when ShipperConfigured=true but ShipperConnected=false. This triggers
the full reconnect protocol (dial + handshake + bounded catch-up)
proactively, bringing the replica current without waiting for I/O.

Option B approach: uses the same reconnect path as Barrier() — not a
fake write or bare dial probe. CatchUpTo(headLSN) replays any retained
WAL entries, bringing the replica fully current.

New methods:
- WALShipper.TryReconnect(): full reconnect without foreground I/O
- ShipperGroup.TryReconnectAll(): probes all disconnected shippers
- BlockVol.TryReconnectShippers(): volume-level entry point

Also fix pre-existing test expectation: engine now emits
start_recovery_task on primary assignment with replicas.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pingqiu
2026-04-09 00:14:46 -07:00
co-authored by Claude Opus 4.6
parent 53246d2780
commit f90ccf5bfd
5 changed files with 103 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/server/volume_server_block.go
+16
View File
@@ -1332,6 +1332,22 @@ func (bs *BlockService) observePrimaryShipperConnectivity(path string) {
return
}
connected := bs.isPrimaryShipperConnected(path)
if !connected {
// Proactive reconnect: the shipper is configured but not connected,
// and no I/O is happening to trigger Ship(). This occurs on rejoin
// paths where the primary gets a fresh assignment with replica
// addresses but no writes are pending. Without this, the shipper
// sits at Disconnected and the core stays at
// awaiting_shipper_connected indefinitely.
//
// Uses the full reconnect protocol (handshake + bounded catch-up),
// not just a dial probe. This brings the replica current if WAL
// entries are available.
_ = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error {
connected = vol.TryReconnectShippers()
return nil
})
}
glog.V(0).Infof("block service: recheck shipper connectivity %s connected=%v mode=%s reason=%q",
path, connected, proj.Mode.Name, proj.Publication.Reason)
bs.observePrimaryShipperConnectivityStatus(path, connected)
weed/server/volume_server_block_test.go
+8 -3
View File
@@ -588,7 +588,10 @@ func TestBlockService_ApplyAssignments_ExecutesCoreCommands_PrimaryRoleApplyAndC
if status.Role != blockvol.RolePrimary || status.Epoch != 1 {
t.Fatalf("status=%+v", status)
}
if got := bs.ExecutedCoreCommands(path); !reflect.DeepEqual(got, []string{"apply_role", "configure_shipper"}) {
got := bs.ExecutedCoreCommands(path)
// Engine now also emits start_recovery_task for primary assignments with replicas.
if !reflect.DeepEqual(got, []string{"apply_role", "configure_shipper"}) &&
!reflect.DeepEqual(got, []string{"apply_role", "configure_shipper", "start_recovery_task"}) {
t.Fatalf("executed commands=%v", got)
}
@@ -596,8 +599,10 @@ func TestBlockService_ApplyAssignments_ExecutesCoreCommands_PrimaryRoleApplyAndC
if len(errs) != 1 || errs[0] != nil {
t.Fatalf("second apply errs=%v", errs)
}
if got := bs.ExecutedCoreCommands(path); !reflect.DeepEqual(got, []string{"apply_role", "configure_shipper"}) {
t.Fatalf("unchanged assignment should not re-execute command chain, got %v", got)
got2 := bs.ExecutedCoreCommands(path)
// Second apply should not add new commands beyond what the first produced.
if !reflect.DeepEqual(got2, got) {
t.Fatalf("unchanged assignment should not re-execute command chain, first=%v second=%v", got, got2)
}
}