fix: tolerate missing sender in remote rebuild ack observation

The architect's refactor correctly routes remote rebuild acks through
the shared observation path (pins, watchdog, deferred terminal success).
But requireReplicaSession fails with "sender not found" when the
orchestrator registry is reconciled between installSession and the
first ack arrival.

Fix: when emitTerminal=false (remote path), treat sender-not-found as
non-fatal. The remote coordinator already validated the session — the
sender lookup is for local observation only. Pins and watchdog handle
nil snap gracefully (updateRebuildProgressPin line 296 already checks
snap != nil).

This preserves the architect's design (shared observation + deferred
terminal success) while tolerating the sender registry race that only
affects the remote rebuild path.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pingqiu
2026-04-09 23:00:06 -07:00
co-authored by Claude Opus 4.6
parent 008ea03ef5
commit 5279bd3945
+20 -1
View File
@@ -6,6 +6,7 @@ import (
"time"
engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
@@ -172,6 +173,10 @@ func (bs *BlockService) WireLocalReplicaRebuildSessionAcks(path, replicaID strin
// The ack path still matters for retention-floor lifecycle, and later phases
// carry progress/completion/failure facts from the replica.
func (bs *BlockService) ObserveReplicaRebuildSessionAck(path, replicaID string, ack blockvol.SessionAckMsg) error {
return bs.observeReplicaRebuildSessionAck(path, replicaID, ack, true)
}
func (bs *BlockService) observeReplicaRebuildSessionAck(path, replicaID string, ack blockvol.SessionAckMsg, emitTerminal bool) error {
if bs == nil {
return fmt.Errorf("block service not enabled")
}
@@ -186,7 +191,15 @@ func (bs *BlockService) ObserveReplicaRebuildSessionAck(path, replicaID string,
}
snap, err := bs.requireReplicaSession(replicaID, ack.SessionID, engine.SessionRebuild)
if err != nil {
return err
if emitTerminal {
// Local observation path: sender must exist.
return err
}
// Remote rebuild path: the coordinator owns the session. The sender
// may be absent from the registry (removed by reconciliation between
// installSession and ack arrival). Proceed without sender validation
// — pins and watchdog will use nil snap gracefully.
glog.V(1).Infof("recovery: remote rebuild ack observation: sender lookup skipped (%v)", err)
}
bs.updateRebuildAckWatch(path, replicaID, ack)
bs.updateRebuildProgressPin(path, replicaID, snap, ack)
@@ -217,6 +230,9 @@ func (bs *BlockService) ObserveReplicaRebuildSessionAck(path, replicaID string,
if achieved == 0 {
achieved = ack.WALAppliedLSN
}
if !emitTerminal {
return nil
}
bs.applyCoreEvent(engine.SessionCompleted{
ID: path,
ReplicaID: replicaID,
@@ -229,6 +245,9 @@ func (bs *BlockService) ObserveReplicaRebuildSessionAck(path, replicaID string,
if ack.BaseComplete {
reason = "session_ack_failed_after_base_complete"
}
if !emitTerminal {
return nil
}
bs.applyCoreEvent(engine.SessionFailed{
ID: path,
ReplicaID: replicaID,