From 5279bd39452e7dbf871adc7f5133af8a1d884ca7 Mon Sep 17 00:00:00 2001 From: pingqiu Date: Thu, 9 Apr 2026 23:00:06 -0700 Subject: [PATCH] fix: tolerate missing sender in remote rebuild ack observation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The architect's refactor correctly routes remote rebuild acks through the shared observation path (pins, watchdog, deferred terminal success). But requireReplicaSession fails with "sender not found" when the orchestrator registry is reconciled between installSession and the first ack arrival. Fix: when emitTerminal=false (remote path), treat sender-not-found as non-fatal. The remote coordinator already validated the session — the sender lookup is for local observation only. Pins and watchdog handle nil snap gracefully (updateRebuildProgressPin line 296 already checks snap != nil). This preserves the architect's design (shared observation + deferred terminal success) while tolerating the sender registry race that only affects the remote rebuild path. Co-Authored-By: Claude Opus 4.6 (1M context) --- weed/server/block_rebuild_session.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/weed/server/block_rebuild_session.go b/weed/server/block_rebuild_session.go index bfeea176a..a06ea51c0 100644 --- a/weed/server/block_rebuild_session.go +++ b/weed/server/block_rebuild_session.go @@ -6,6 +6,7 @@ import ( "time" engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" ) @@ -172,6 +173,10 @@ func (bs *BlockService) WireLocalReplicaRebuildSessionAcks(path, replicaID strin // The ack path still matters for retention-floor lifecycle, and later phases // carry progress/completion/failure facts from the replica. func (bs *BlockService) ObserveReplicaRebuildSessionAck(path, replicaID string, ack blockvol.SessionAckMsg) error { + return bs.observeReplicaRebuildSessionAck(path, replicaID, ack, true) +} + +func (bs *BlockService) observeReplicaRebuildSessionAck(path, replicaID string, ack blockvol.SessionAckMsg, emitTerminal bool) error { if bs == nil { return fmt.Errorf("block service not enabled") } @@ -186,7 +191,15 @@ func (bs *BlockService) ObserveReplicaRebuildSessionAck(path, replicaID string, } snap, err := bs.requireReplicaSession(replicaID, ack.SessionID, engine.SessionRebuild) if err != nil { - return err + if emitTerminal { + // Local observation path: sender must exist. + return err + } + // Remote rebuild path: the coordinator owns the session. The sender + // may be absent from the registry (removed by reconciliation between + // installSession and ack arrival). Proceed without sender validation + // — pins and watchdog will use nil snap gracefully. + glog.V(1).Infof("recovery: remote rebuild ack observation: sender lookup skipped (%v)", err) } bs.updateRebuildAckWatch(path, replicaID, ack) bs.updateRebuildProgressPin(path, replicaID, snap, ack) @@ -217,6 +230,9 @@ func (bs *BlockService) ObserveReplicaRebuildSessionAck(path, replicaID string, if achieved == 0 { achieved = ack.WALAppliedLSN } + if !emitTerminal { + return nil + } bs.applyCoreEvent(engine.SessionCompleted{ ID: path, ReplicaID: replicaID, @@ -229,6 +245,9 @@ func (bs *BlockService) ObserveReplicaRebuildSessionAck(path, replicaID string, if ack.BaseComplete { reason = "session_ack_failed_after_base_complete" } + if !emitTerminal { + return nil + } bs.applyCoreEvent(engine.SessionFailed{ ID: path, ReplicaID: replicaID,