seaweedfs/weed/storage/blockvol/wal_shipper.go

package blockvol

import (
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"sync/atomic"
	"time"
)

var (
	ErrReplicaDegraded = errors.New("blockvol: replica degraded")
	ErrShipperStopped  = errors.New("blockvol: shipper stopped")
)

const barrierTimeout = 5 * time.Second

// ReplicaState tracks the replication state machine for one replica.
// Only InSync replicas are eligible for sync_all barrier participation.
type ReplicaState uint32

const (
	ReplicaDisconnected ReplicaState = 0 // no session (initial state)
	ReplicaConnecting   ReplicaState = 1 // socket open, handshake pending (CP13-5)
	ReplicaCatchingUp   ReplicaState = 2 // connected, replaying missed WAL (CP13-5)
	ReplicaInSync       ReplicaState = 3 // eligible for sync_all barriers
	ReplicaDegraded     ReplicaState = 4 // transient failure, retry allowed
	ReplicaNeedsRebuild ReplicaState = 5 // WAL gap too large, rebuild required (CP13-7)
)

func (s ReplicaState) String() string {
	switch s {
	case ReplicaDisconnected:
		return "disconnected"
	case ReplicaConnecting:
		return "connecting"
	case ReplicaCatchingUp:
		return "catching_up"
	case ReplicaInSync:
		return "in_sync"
	case ReplicaDegraded:
		return "degraded"
	case ReplicaNeedsRebuild:
		return "needs_rebuild"
	default:
		return fmt.Sprintf("unknown(%d)", s)
	}
}

// WALShipper streams WAL entries from the primary to a replica over TCP.
// Fire-and-forget: no per-entry ACK. Barriers provide durability confirmation.
type WALShipper struct {
	dataAddr    string
	controlAddr string
	epochFn     func() uint64
	wal         WALAccess      // primary WAL access for reconnect catch-up
	metrics     *EngineMetrics

	mu       sync.Mutex // protects dataConn
	dataConn net.Conn

	ctrlMu   sync.Mutex // protects ctrlConn
	ctrlConn net.Conn

	shippedLSN         atomic.Uint64 // diagnostic: highest LSN sent to TCP socket
	replicaFlushedLSN  atomic.Uint64 // authoritative: highest LSN durably persisted on replica
	hasFlushedProgress atomic.Bool   // true once replica returns a valid (non-zero) FlushedLSN
	state              atomic.Uint32 // ReplicaState
	catchupFailures    int           // consecutive catch-up failures; reset on success
	lastContactTime    atomic.Value  // time.Time: last successful barrier/handshake/catch-up
	stopped            atomic.Bool

	// onStateChange is called when the shipper transitions between states.
	// Used to trigger immediate heartbeat on degradation/recovery.
	// Set via SetOnStateChange. Nil = no callback.
	onStateChange func(from, to ReplicaState)
}

// SetOnStateChange registers a callback for shipper state transitions.
// The callback is invoked synchronously from markDegraded/markInSync.
func (s *WALShipper) SetOnStateChange(fn func(from, to ReplicaState)) {
	s.onStateChange = fn
}

const maxCatchupRetries = 3

// NewWALShipper creates a WAL shipper. Connections are established lazily on
// first Ship/Barrier call. epochFn returns the current epoch for validation.
// wal provides WAL access for reconnect catch-up (nil disables catch-up).
// metrics is optional; if nil, no metrics are recorded.
func NewWALShipper(dataAddr, controlAddr string, epochFn func() uint64, walAccess WALAccess, metrics ...*EngineMetrics) *WALShipper {
	var m *EngineMetrics
	if len(metrics) > 0 {
		m = metrics[0]
	}
	return &WALShipper{
		dataAddr:    dataAddr,
		controlAddr: controlAddr,
		epochFn:     epochFn,
		wal:         walAccess,
		metrics:     m,
	}
}

// Ship sends a WAL entry to the replica over the data channel.
// On write error, the shipper enters degraded mode. Recovery requires
// the full reconnect protocol. See design/sync-all-reconnect-protocol.md.
func (s *WALShipper) Ship(entry *WALEntry) error {
	st := s.State()
	// Ship allowed from Disconnected (bootstrap: data must flow before first barrier)
	// and InSync (steady state). All other states reject.
	if s.stopped.Load() || (st != ReplicaInSync && st != ReplicaDisconnected) {
		return nil
	}

	// Validate epoch: drop stale entries.
	if entry.Epoch != s.epochFn() {
		log.Printf("wal_shipper: dropping entry LSN=%d with stale epoch %d (current %d)",
			entry.LSN, entry.Epoch, s.epochFn())
		return nil
	}

	encoded, err := entry.Encode()
	if err != nil {
		return fmt.Errorf("wal_shipper: encode entry: %w", err)
	}

	s.mu.Lock()
	defer s.mu.Unlock()

	if err := s.ensureDataConn(); err != nil {
		s.markDegraded()
		return nil
	}

	// Set a write deadline so we don't block for the full TCP
	// retransmission timeout (~120s) if the replica is dead.
	s.dataConn.SetWriteDeadline(time.Now().Add(3 * time.Second))
	if err := WriteFrame(s.dataConn, MsgWALEntry, encoded); err != nil {
		s.dataConn.SetWriteDeadline(time.Time{}) // clear deadline
		s.markDegraded()
		return nil
	}
	s.dataConn.SetWriteDeadline(time.Time{}) // clear deadline

	s.shippedLSN.Store(entry.LSN)
	if s.metrics != nil {
		s.metrics.RecordWALShipped()
	}
	return nil
}

// Barrier sends a barrier request on the control channel and waits for the
// replica to confirm durability up to lsnMax. Returns ErrReplicaDegraded if
// the shipper is in degraded mode. Reconnection requires the full reconnect
// protocol (ResumeShipReq handshake + WAL catch-up), not just TCP retry.
// See design/sync-all-reconnect-protocol.md.
func (s *WALShipper) Barrier(lsnMax uint64) error {
	if s.stopped.Load() {
		return ErrShipperStopped
	}

	st := s.State()
	switch st {
	case ReplicaInSync:
		// proceed normally to barrier
	case ReplicaDisconnected, ReplicaDegraded:
		if s.hasFlushedProgress.Load() && s.wal != nil {
			// Previously synced — reconnect handshake + catch-up path.
			if err := s.doReconnectAndCatchUp(); err != nil {
				return err
			}
		} else {
			// Fresh bootstrap or no WAL access — reset connections for bare retry.
			s.resetConnections()
		}
	default:
		// Connecting, CatchingUp, NeedsRebuild — reject immediately
		return ErrReplicaDegraded
	}

	barrierStart := time.Now()

	req := EncodeBarrierRequest(BarrierRequest{
		LSN:   lsnMax,
		Epoch: s.epochFn(),
	})

	s.ctrlMu.Lock()
	defer s.ctrlMu.Unlock()

	if err := s.ensureCtrlConn(); err != nil {
		s.markDegraded()
		s.recordBarrierMetric(barrierStart, true)
		return ErrReplicaDegraded
	}

	s.ctrlConn.SetDeadline(time.Now().Add(barrierTimeout))

	if err := WriteFrame(s.ctrlConn, MsgBarrierReq, req); err != nil {
		s.markDegraded()
		s.recordBarrierMetric(barrierStart, true)
		return ErrReplicaDegraded
	}

	msgType, payload, err := ReadFrame(s.ctrlConn)
	if err != nil {
		s.markDegraded()
		s.recordBarrierMetric(barrierStart, true)
		return ErrReplicaDegraded
	}

	if msgType != MsgBarrierResp || len(payload) < 1 {
		s.markDegraded()
		s.recordBarrierMetric(barrierStart, true)
		return ErrReplicaDegraded
	}

	resp := DecodeBarrierResponse(payload)

	switch resp.Status {
	case BarrierOK:
		// CP13-3: BarrierOK with FlushedLSN == 0 means the replica confirmed
		// receipt + fsync but did not report which LSN is durable (legacy 1-byte
		// response). This must NOT count as successful sync_all durability because
		// no authoritative durable progress was established.
		if resp.FlushedLSN == 0 {
			s.recordBarrierMetric(barrierStart, true)
			return fmt.Errorf("wal_shipper: barrier OK but no FlushedLSN reported (legacy response)")
		}
		// Barrier success with durable progress — transition to InSync.
		s.markInSync()
		// Update authoritative durable progress (monotonic: only advance).
		s.hasFlushedProgress.Store(true)
		for {
			cur := s.replicaFlushedLSN.Load()
			if resp.FlushedLSN <= cur {
				break
			}
			if s.replicaFlushedLSN.CompareAndSwap(cur, resp.FlushedLSN) {
				break
			}
		}
		s.recordBarrierMetric(barrierStart, false)
		return nil
	case BarrierEpochMismatch:
		s.markDegraded()
		s.recordBarrierMetric(barrierStart, true)
		return fmt.Errorf("wal_shipper: barrier epoch mismatch")
	case BarrierTimeout:
		s.markDegraded()
		s.recordBarrierMetric(barrierStart, true)
		return fmt.Errorf("wal_shipper: barrier timeout on replica")
	case BarrierFsyncFailed:
		s.markDegraded()
		s.recordBarrierMetric(barrierStart, true)
		return fmt.Errorf("wal_shipper: barrier fsync failed on replica")
	default:
		s.markDegraded()
		s.recordBarrierMetric(barrierStart, true)
		return fmt.Errorf("wal_shipper: unknown barrier status %d", payload[0])
	}
}

func (s *WALShipper) recordBarrierMetric(start time.Time, failed bool) {
	if s.metrics != nil {
		s.metrics.RecordWALBarrier(time.Since(start), failed)
	}
}

// ShippedLSN returns the highest LSN successfully sent to the replica (diagnostic only).
// This is NOT authoritative for sync durability — use ReplicaFlushedLSN() instead.
func (s *WALShipper) ShippedLSN() uint64 {
	return s.shippedLSN.Load()
}

// ReplicaFlushedLSN returns the highest LSN durably persisted on the replica,
// as reported in the barrier response after fd.Sync(). This is the authoritative
// durable progress variable for sync_all correctness.
func (s *WALShipper) ReplicaFlushedLSN() uint64 {
	return s.replicaFlushedLSN.Load()
}

// HasFlushedProgress returns true if the replica has ever reported a valid
// (non-zero) FlushedLSN. Legacy replicas that only support 1-byte barrier
// responses will never set this, and must not count toward sync_all.
func (s *WALShipper) HasFlushedProgress() bool {
	return s.hasFlushedProgress.Load()
}

// State returns the current replica state machine state.
func (s *WALShipper) State() ReplicaState {
	return ReplicaState(s.state.Load())
}

// LastContactTime returns the last time this replica had successful
// durable contact (barrier success, reconnect handshake, catch-up completion).
// Returns zero time if no contact has occurred.
func (s *WALShipper) LastContactTime() time.Time {
	if v := s.lastContactTime.Load(); v != nil {
		return v.(time.Time)
	}
	return time.Time{}
}

func (s *WALShipper) touchContactTime() {
	s.lastContactTime.Store(time.Now())
}

// IsDegraded returns true if the replica is not sync-eligible (any state
// other than InSync). This overloads Disconnected, Connecting, CatchingUp,
// NeedsRebuild, and Degraded into one "not healthy" shape for backward
// compatibility with existing metrics and callers.
func (s *WALShipper) IsDegraded() bool {
	return s.State() != ReplicaInSync
}

// Stop shuts down the shipper and closes connections.
func (s *WALShipper) Stop() {
	if s.stopped.Swap(true) {
		return
	}
	s.mu.Lock()
	if s.dataConn != nil {
		s.dataConn.Close()
		s.dataConn = nil
	}
	s.mu.Unlock()

	s.ctrlMu.Lock()
	if s.ctrlConn != nil {
		s.ctrlConn.Close()
		s.ctrlConn = nil
	}
	s.ctrlMu.Unlock()
}

func (s *WALShipper) ensureDataConn() error {
	if s.dataConn != nil {
		return nil
	}
	conn, err := net.DialTimeout("tcp", s.dataAddr, 3*time.Second)
	if err != nil {
		return err
	}
	s.dataConn = conn
	return nil
}

func (s *WALShipper) ensureCtrlConn() error {
	if s.ctrlConn != nil {
		return nil
	}
	conn, err := net.DialTimeout("tcp", s.controlAddr, 3*time.Second)
	if err != nil {
		return err
	}
	s.ctrlConn = conn
	return nil
}

func (s *WALShipper) markDegraded() {
	prev := ReplicaState(s.state.Swap(uint32(ReplicaDegraded)))
	log.Printf("wal_shipper: replica degraded (data=%s, ctrl=%s, prev=%s)", s.dataAddr, s.controlAddr, prev)
	if prev != ReplicaDegraded && s.onStateChange != nil {
		s.onStateChange(prev, ReplicaDegraded)
	}
}

// resetConnections closes both data and control connections for a clean retry.
func (s *WALShipper) resetConnections() {
	s.mu.Lock()
	if s.dataConn != nil {
		s.dataConn.Close()
		s.dataConn = nil
	}
	s.mu.Unlock()
	s.ctrlMu.Lock()
	if s.ctrlConn != nil {
		s.ctrlConn.Close()
		s.ctrlConn = nil
	}
	s.ctrlMu.Unlock()
}

// doReconnectAndCatchUp runs the full reconnect handshake + catch-up protocol.
// On success, transitions to InSync and resets ctrl connection for barrier.
func (s *WALShipper) doReconnectAndCatchUp() error {
	targetState, replicaFlushed, err := s.reconnectWithHandshake()
	switch targetState {
	case ReplicaInSync:
		s.markInSync()
	case ReplicaCatchingUp:
		// Use the handshake-reported flushedLSN as catch-up start point,
		// NOT the shipper's cached value. The replica may have lost progress
		// since the shipper last heard from it.
		if catchErr := s.runCatchUp(replicaFlushed); catchErr != nil {
			s.catchupFailures++
			if s.catchupFailures >= maxCatchupRetries {
				s.state.Store(uint32(ReplicaNeedsRebuild))
				return fmt.Errorf("catch-up failed %d times: %w", s.catchupFailures, catchErr)
			}
			s.markDegraded()
			return ErrReplicaDegraded
		}
		s.markInSync()
	case ReplicaNeedsRebuild:
		s.state.Store(uint32(ReplicaNeedsRebuild))
		return fmt.Errorf("reconnect: %w", err)
	default:
		s.markDegraded()
		return ErrReplicaDegraded
	}
	// Reset ctrl connection so barrier creates a fresh one.
	s.ctrlMu.Lock()
	if s.ctrlConn != nil {
		s.ctrlConn.Close()
		s.ctrlConn = nil
	}
	s.ctrlMu.Unlock()
	return nil
}

func (s *WALShipper) markInSync() {
	prev := ReplicaState(s.state.Swap(uint32(ReplicaInSync)))
	s.catchupFailures = 0
	s.touchContactTime()
	log.Printf("wal_shipper: replica in-sync (data=%s, ctrl=%s, prev=%s)", s.dataAddr, s.controlAddr, prev)
	if prev != ReplicaInSync && s.onStateChange != nil {
		s.onStateChange(prev, ReplicaInSync)
	}
}

const catchupTimeout = 30 * time.Second

// reconnectWithHandshake performs the CP13-5 reconnect protocol:
// connect data channel → send ResumeShipReq → read ResumeShipResp → decide.
// Returns the target state (InSync, CatchingUp, NeedsRebuild) and replica's flushed LSN.
// Caller must hold no locks. Must only be called when wal != nil.
func (s *WALShipper) reconnectWithHandshake() (targetState ReplicaState, replicaFlushedLSN uint64, err error) {
	s.state.Store(uint32(ReplicaConnecting))

	// Reset and establish data connection.
	s.mu.Lock()
	if s.dataConn != nil {
		s.dataConn.Close()
		s.dataConn = nil
	}
	if err := s.ensureDataConn(); err != nil {
		s.mu.Unlock()
		return ReplicaDegraded, 0, fmt.Errorf("reconnect dial: %w", err)
	}
	s.dataConn.SetDeadline(time.Now().Add(catchupTimeout))
	conn := s.dataConn
	s.mu.Unlock()

	// Gather primary state.
	retainStart, headLSN := s.wal.RetainedRange()
	epoch := s.epochFn()

	// Send ResumeShipReq.
	req := EncodeResumeShipReq(ResumeShipReq{
		Epoch:          epoch,
		PrimaryHeadLSN: headLSN,
		WalRetainStart: retainStart,
	})
	if err := WriteFrame(conn, MsgResumeShipReq, req); err != nil {
		return ReplicaDegraded, 0, fmt.Errorf("reconnect send req: %w", err)
	}

	// Read ResumeShipResp.
	msgType, payload, err := ReadFrame(conn)
	if err != nil {
		return ReplicaDegraded, 0, fmt.Errorf("reconnect read resp: %w", err)
	}
	if msgType != MsgResumeShipResp {
		return ReplicaDegraded, 0, fmt.Errorf("reconnect: unexpected msg type 0x%02x", msgType)
	}
	resp, err := DecodeResumeShipResp(payload)
	if err != nil {
		return ReplicaDegraded, 0, err
	}

	// Clear deadline for catch-up streaming.
	s.mu.Lock()
	if s.dataConn != nil {
		s.dataConn.SetDeadline(time.Time{})
	}
	s.mu.Unlock()

	// Decision matrix.
	switch resp.Status {
	case ResumeEpochMismatch:
		return ReplicaNeedsRebuild, resp.ReplicaFlushedLSN, fmt.Errorf("reconnect: epoch mismatch")
	case ResumeNeedsRebuild:
		return ReplicaNeedsRebuild, resp.ReplicaFlushedLSN, fmt.Errorf("reconnect: replica requests rebuild")
	case ResumeOK:
		// proceed to gap analysis
	default:
		return ReplicaDegraded, resp.ReplicaFlushedLSN, fmt.Errorf("reconnect: unknown status 0x%02x", resp.Status)
	}

	R := resp.ReplicaFlushedLSN
	H := headLSN
	S := retainStart

	if R > H {
		// Impossible: replica ahead of primary.
		log.Printf("wal_shipper: reconnect %s: impossible progress R=%d > H=%d", s.dataAddr, R, H)
		return ReplicaNeedsRebuild, R, fmt.Errorf("reconnect: impossible replica progress")
	}
	if R == H {
		// Already caught up.
		log.Printf("wal_shipper: reconnect %s: already caught up (R=H=%d)", s.dataAddr, R)
		return ReplicaInSync, R, nil
	}
	if R+1 >= S {
		// Recoverable gap: WAL still has entries from R+1.
		log.Printf("wal_shipper: reconnect %s: recoverable gap R=%d H=%d S=%d", s.dataAddr, R, H, S)
		return ReplicaCatchingUp, R, nil
	}
	// Gap exceeds retained WAL.
	log.Printf("wal_shipper: reconnect %s: gap too large R=%d H=%d S=%d", s.dataAddr, R, H, S)
	return ReplicaNeedsRebuild, R, fmt.Errorf("reconnect: gap exceeds retained WAL")
}

// runCatchUp streams WAL entries from fromLSN+1 to the replica on the data channel.
// Sends MsgCatchupDone when complete. Caller must hold no shipper locks.
func (s *WALShipper) runCatchUp(fromLSN uint64) error {
	s.state.Store(uint32(ReplicaCatchingUp))

	// Set a deadline for the entire catch-up operation.
	s.mu.Lock()
	if s.dataConn != nil {
		s.dataConn.SetDeadline(time.Now().Add(catchupTimeout))
	}
	conn := s.dataConn
	s.mu.Unlock()

	if conn == nil {
		return fmt.Errorf("catch-up: no data connection")
	}

	// Stream entries from WAL.
	var lastSent uint64
	err := s.wal.StreamEntries(fromLSN+1, func(entry *WALEntry) error {
		encoded, encErr := entry.Encode()
		if encErr != nil {
			return encErr
		}
		if wErr := WriteFrame(conn, MsgWALEntry, encoded); wErr != nil {
			return wErr
		}
		lastSent = entry.LSN
		return nil
	})

	if err != nil {
		if errors.Is(err, ErrWALRecycled) {
			s.state.Store(uint32(ReplicaNeedsRebuild))
			return fmt.Errorf("catch-up: WAL recycled: %w", err)
		}
		return fmt.Errorf("catch-up: stream error: %w", err)
	}

	// Send CatchupDone marker.
	_, headLSN := s.wal.RetainedRange()
	if err := WriteFrame(conn, MsgCatchupDone, EncodeCatchupDone(headLSN)); err != nil {
		return fmt.Errorf("catch-up: send done: %w", err)
	}

	// Clear deadline.
	s.mu.Lock()
	if s.dataConn != nil {
		s.dataConn.SetDeadline(time.Time{})
	}
	s.mu.Unlock()

	log.Printf("wal_shipper: catch-up complete %s: from=%d last=%d", s.dataAddr, fromLSN+1, lastSent)
	return nil
}