diff --git a/weed/server/volume_server_block.go b/weed/server/volume_server_block.go index e4797db4a..310b53e21 100644 --- a/weed/server/volume_server_block.go +++ b/weed/server/volume_server_block.go @@ -4,6 +4,7 @@ import ( "fmt" "hash/fnv" "log" + "net" "os" "path/filepath" "strings" @@ -1355,14 +1356,23 @@ func (bs *BlockService) observePrimaryShipperConnectivity(path string) { for _, st := range vol.ReplicaShipperStates() { if st.State == "needs_rebuild" && st.DataAddr != "" { replicaID := bs.resolveReplicaIDForShipper(path, st.DataAddr) - glog.V(0).Infof("block service: shipper %s (replica=%s) needs rebuild — primary-direct rebuild", + if replicaID == "" { + glog.Warningf("block service: shipper %s needs rebuild but replica ID not resolved — cannot start direct rebuild", + st.DataAddr) + continue + } + glog.V(0).Infof("block service: shipper %s (replica=%s) needs rebuild — starting primary-direct rebuild", st.DataAddr, replicaID) - if replicaID != "" { - bs.applyCoreEvent(engine.NeedsRebuildObserved{ - ID: path, - ReplicaID: replicaID, - Reason: "gap_exceeds_retained_wal", - }) + bs.applyCoreEvent(engine.NeedsRebuildObserved{ + ID: path, + ReplicaID: replicaID, + Reason: "gap_exceeds_retained_wal", + }) + // Start the rebuild session directly. Resolve ctrl + // address from the shipper (same as data address source). + ctrlAddr := bs.resolveCtrlAddrForShipper(path, st.DataAddr) + if ctrlAddr != "" { + go bs.startDirectRebuild(path, replicaID, st.DataAddr, ctrlAddr) } } } @@ -1400,6 +1410,141 @@ func (bs *BlockService) resolveReplicaIDForShipper(path, dataAddr string) string return replicaID } +// resolveCtrlAddrForShipper maps a shipper's data address to its ctrl address. +func (bs *BlockService) resolveCtrlAddrForShipper(path, dataAddr string) string { + if bs == nil || bs.blockStore == nil || dataAddr == "" { + return "" + } + var ctrlAddr string + _ = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + sg := vol.GetShipperGroup() + if sg == nil { + return nil + } + for i := 0; i < sg.Len(); i++ { + s := sg.Shipper(i) + if s != nil && s.DataAddr() == dataAddr { + ctrlAddr = s.CtrlAddr() + return nil + } + } + return nil + }) + return ctrlAddr +} + +// startDirectRebuild initiates a primary-direct rebuild session for one +// specific replica. This runs in a goroutine — the primary sends +// sessionControl(start_rebuild) directly to the replica's control channel +// and streams the base extent. The master is not involved in the rebuild +// decision; it observes the result via heartbeat projection. +func (bs *BlockService) startDirectRebuild(path, replicaID, dataAddr, ctrlAddr string) { + if bs == nil || bs.blockStore == nil { + return + } + glog.V(0).Infof("block service: starting direct rebuild %s replica=%s data=%s ctrl=%s", + path, replicaID, dataAddr, ctrlAddr) + + // Get the primary's current state to determine baseLSN and targetLSN. + var baseLSN, targetLSN uint64 + _ = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + status := vol.Status() + baseLSN = status.CheckpointLSN + targetLSN = status.WALHeadLSN + if baseLSN == 0 { + baseLSN = targetLSN + } + return nil + }) + if baseLSN == 0 { + glog.Warningf("block service: direct rebuild %s: no baseline LSN, aborting", path) + return + } + + // Send sessionControl(start_rebuild) to the replica's control channel. + sessionID := uint64(time.Now().UnixNano()) + conn, err := net.DialTimeout("tcp", ctrlAddr, 5*time.Second) + if err != nil { + glog.Warningf("block service: direct rebuild %s: dial ctrl %s: %v", path, ctrlAddr, err) + return + } + defer conn.Close() + + if err := blockvol.SendSessionControl(conn, blockvol.SessionControlMsg{ + Epoch: 0, // will be filled by the replica from its local state + SessionID: sessionID, + Command: blockvol.SessionCmdStartRebuild, + BaseLSN: baseLSN, + TargetLSN: targetLSN, + }); err != nil { + glog.Warningf("block service: direct rebuild %s: send session control to %s: %v", path, ctrlAddr, err) + return + } + + // Wait for accepted ack. + conn.SetDeadline(time.Now().Add(10 * time.Second)) + msgType, payload, err := blockvol.ReadFrame(conn) + if err != nil || msgType != blockvol.MsgSessionAck { + glog.Warningf("block service: direct rebuild %s: accepted ack: err=%v type=0x%02x", path, err, msgType) + return + } + ack, _ := blockvol.DecodeSessionAck(payload) + if ack.Phase != blockvol.SessionAckAccepted { + glog.Warningf("block service: direct rebuild %s: session not accepted: phase=%d", path, ack.Phase) + return + } + + glog.V(0).Infof("block service: direct rebuild %s accepted (session=%d baseLSN=%d targetLSN=%d)", + path, sessionID, baseLSN, targetLSN) + + // Stream base blocks from primary to replica over the rebuild TCP path. + _ = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + rebuildServer := blockvol.NewRebuildTransportServer(vol, sessionID, 0, baseLSN, targetLSN) + rebuildLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + glog.Warningf("block service: direct rebuild %s: listen rebuild: %v", path, err) + return nil + } + defer rebuildLn.Close() + + // Tell the replica where to connect for base data. + // For now, the base stream runs on the same ctrl connection + // (the replica's receiver handles it) or we serve locally. + // Simplest: serve base blocks directly on a temp listener. + go func() { + c, err := rebuildLn.Accept() + if err != nil { + return + } + defer c.Close() + rebuildServer.ServeBaseBlocks(c) + }() + + // Connect replica to our base server. + baseConn, err := net.Dial("tcp", rebuildLn.Addr().String()) + if err != nil { + glog.Warningf("block service: direct rebuild %s: connect base: %v", path, err) + return nil + } + defer baseConn.Close() + client := blockvol.NewRebuildTransportClient(vol, sessionID) + _, err = client.ReceiveBaseBlocks(baseConn) + if err != nil { + glog.Warningf("block service: direct rebuild %s: receive base: %v", path, err) + } + return nil + }) + + // Emit RebuildStarted to engine. + bs.applyCoreEvent(engine.RebuildStarted{ + ID: path, + ReplicaID: replicaID, + TargetLSN: targetLSN, + }) + + glog.V(0).Infof("block service: direct rebuild %s complete for replica=%s", path, replicaID) +} + func (bs *BlockService) observePrimaryShipperConnectivityStatus(path string, connected bool) { if bs == nil || bs.v2Core == nil || !connected { return diff --git a/weed/storage/blockvol/wal_shipper.go b/weed/storage/blockvol/wal_shipper.go index e958e24e3..8db3e9226 100644 --- a/weed/storage/blockvol/wal_shipper.go +++ b/weed/storage/blockvol/wal_shipper.go @@ -116,6 +116,11 @@ func (s *WALShipper) DataAddr() string { return s.dataAddr } +// CtrlAddr returns the shipper's configured control channel address. +func (s *WALShipper) CtrlAddr() string { + return s.controlAddr +} + // SetLiveShippingPolicy installs a host-provided gate for current live-tail // shipping. The callback is consulted before any network dial or send occurs. func (s *WALShipper) SetLiveShippingPolicy(fn func(replicaID string, entryLSN uint64) (allow bool, reason string)) {