seaweedfs/weed/storage/blockvol/dist_group_commit.go

package blockvol

import (
	"fmt"
	"sync"
)

// MakeDistributedSync creates a sync function that runs local WAL fsync and
// replica barriers in parallel. Supports N replicas via ShipperGroup.
//
// Durability semantics depend on vol.DurabilityMode():
//   - best_effort: local fsync = ACK; replica failures degrade shippers but never fail writes
//   - sync_all: ALL replica barriers must succeed, else write returns ErrDurabilityBarrierFailed
//   - sync_quorum: quorum (RF/2+1) of nodes must be durable, else ErrDurabilityQuorumLost
func MakeDistributedSync(walSync func() error, group *ShipperGroup, vol *BlockVol) func() error {
	return func() error {
		mode := vol.DurabilityMode()

		if group == nil || group.Len() == 0 {
			// No replicas configured — local sync only.
			return walSync()
		}

		// Note: we always attempt BarrierAll, even when all shippers are
		// Disconnected or Degraded. Barrier() handles bootstrap (Disconnected)
		// and reconnect (Degraded) paths. Only Connecting/CatchingUp/NeedsRebuild
		// are pre-rejected by individual shippers.

		// The highest LSN that needs to be durable is nextLSN-1.
		lsnMax := vol.nextLSN.Load() - 1

		var localErr error
		var barrierErrs []error
		var wg sync.WaitGroup
		wg.Add(2)
		go func() {
			defer wg.Done()
			localErr = walSync()
		}()
		go func() {
			defer wg.Done()
			barrierErrs = group.BarrierAll(lsnMax)
		}()
		wg.Wait()

		if localErr != nil {
			return localErr
		}

		// Count barrier failures and degrade shippers.
		failCount := 0
		for _, err := range barrierErrs {
			if err != nil {
				failCount++
				vol.degradeReplica(err)
			}
		}

		switch mode {
		case DurabilitySyncAll:
			if failCount > 0 {
				if vol.Metrics != nil {
					vol.Metrics.DurabilityBarrierFailedTotal.Add(1)
				}
				return fmt.Errorf("%w: %d of %d barriers failed",
					ErrDurabilityBarrierFailed, failCount, len(barrierErrs))
			}
		case DurabilitySyncQuorum:
			rf := group.Len() + 1 // total nodes including primary
			quorum := rf/2 + 1
			durableNodes := 1 + (len(barrierErrs) - failCount) // primary + successful barriers
			if durableNodes < quorum {
				if vol.Metrics != nil {
					vol.Metrics.DurabilityQuorumLostTotal.Add(1)
				}
				return fmt.Errorf("%w: %d durable of %d needed",
					ErrDurabilityQuorumLost, durableNodes, quorum)
			}
		}
		// best_effort: barrier failures already logged via degradeReplica, return nil.
		return nil
	}
}