fix: three hardware blockers — WAL retention + registry race + shutdown beat

All 43 actions pass on m01/m02 hardware. Auto-failover PASS.
dd_write: 30s → 123ms. Post-failover write: 33,621 IOPS.

1. WAL retention: remove keepup retention floor (MinShippedLSN).
   WAL cannot be pinned during sustained async writes — any pin
   strategy either fills WAL (blocking writes) or over-recycles
   (breaking catch-up). Flusher recycles freely. Future LBA map
   will provide catch-up without WAL retention.
   MinShippedLSN on ShipperGroup retained as diagnostic surface.

2. Registry stale-cleanup race: add RegisteredAt grace period.
   Race: master registers volume → next VS heartbeat arrives before
   VS discovers the volume → stale cleanup deletes the entry →
   failover finds 0 entries. Fix: skip stale cleanup for entries
   registered within 30s (> 2 heartbeat intervals).
   2 new tests: grace protects new entry, old entry still cleaned.

3. Shutdown heartbeat: VS disconnect heartbeat no longer claims
   block inventory authority. Previously, the shutdown beat's
   empty inventory triggered stale cleanup, deleting the entry
   before failover could use it.

Scenario fix: recovery-baseline-failover.yaml now kills the
correct node (discovered primary, not hardcoded), connects to
the correct new primary for post-failover verification.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

weed/server/master_block_failover.go

@@ -196,6 +196,11 @@ func (ms *MasterServer) failoverBlockVolumes(deadServer string) {
 	}
 	ms.blockRegistry.FailoversTotal.Add(1)
 	entries := ms.blockRegistry.ListByServer(deadServer)
+	glog.V(0).Infof("failover: deadServer=%s entries=%d", deadServer, len(entries))
+	for i, e := range entries {
+		glog.V(0).Infof("failover: entry[%d] name=%q vs=%s role=%d hasReplica=%v epoch=%d",
+			i, e.Name, e.VolumeServer, e.Role, e.HasReplica(), e.Epoch)
+	}
 	now := time.Now()
 	for _, entry := range entries {
 		// Case 1: Dead server is the primary.

weed/server/master_block_registry.go

@@ -111,6 +111,12 @@ type BlockVolumeEntry struct {
 	LastLeaseGrant time.Time
 	LeaseTTL       time.Duration
 
+	// Registration race protection: the time this entry was created/registered
+	// by the master. Stale cleanup skips recently registered entries to allow
+	// the volume server time to discover the volume and include it in its
+	// next heartbeat inventory.
+	RegisteredAt time.Time
+
 	// CP11A-2: Coordinated expand tracking.
 	ExpandInProgress  bool
 	ExpandFailed      bool // true = primary committed but replica(s) failed; size suppressed
@@ -424,6 +430,9 @@ func (r *BlockVolumeRegistry) Register(entry *BlockVolumeEntry) error {
 	if _, ok := r.volumes[entry.Name]; ok {
 		return fmt.Errorf("block volume %q already registered", entry.Name)
 	}
+	if entry.RegisteredAt.IsZero() {
+		entry.RegisteredAt = time.Now()
+	}
 	entry.recomputeReplicaState()
 	r.volumes[entry.Name] = entry
 	r.addToServer(entry.VolumeServer, entry.Name)
@@ -642,6 +651,14 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeatWithInventoryAuthority(server s
 								name, server)
 							continue
 						}
+						// Registration race protection: skip recently registered entries.
+						// The VS may not have discovered the volume yet. Grace period
+						// of 30s (> 2 heartbeat intervals) prevents premature deletion.
+						if !entry.RegisteredAt.IsZero() && time.Since(entry.RegisteredAt) < 30*time.Second {
+							glog.V(0).Infof("block registry: skipping stale-cleanup for %q (registered %v ago, grace period)",
+								name, time.Since(entry.RegisteredAt).Round(time.Second))
+							continue
+						}
 						delete(r.volumes, name)
 						delete(names, name)
 						// Also clean up replica entries from byServer.

weed/server/master_block_registry_test.go

@@ -88,8 +88,9 @@ func TestRegistry_ListByServer(t *testing.T) {
 func TestRegistry_UpdateFullHeartbeat(t *testing.T) {
 	r := NewBlockVolumeRegistry()
 	// Register two volumes on server s1.
-	r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusPending})
-	r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s1", Path: "/v2.blk", Status: StatusPending})
+	pastGrace := time.Now().Add(-60 * time.Second)
+	r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusPending, RegisteredAt: pastGrace})
+	r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s1", Path: "/v2.blk", Status: StatusPending, RegisteredAt: pastGrace})
 
 	// Full heartbeat reports only vol1 (vol2 is stale).
 	r.UpdateFullHeartbeat("s1", []*master_pb.BlockVolumeInfoMessage{
@@ -127,7 +128,7 @@ func TestRegistry_UpdateFullHeartbeatWithInventoryAuthority_NonAuthoritativeEmpt
 
 func TestRegistry_UpdateFullHeartbeatWithInventoryAuthority_AuthoritativeEmptyStillDeletes(t *testing.T) {
 	r := NewBlockVolumeRegistry()
-	r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusActive})
+	r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusActive, RegisteredAt: time.Now().Add(-60 * time.Second)})
 
 	r.UpdateFullHeartbeatWithInventoryAuthority("s1", nil, "", true)
 
@@ -3206,3 +3207,58 @@ func TestRegistry_UpdateFullHeartbeat_EngineProjectionModePreservedOnNewPrimaryW
 		t.Fatalf("EngineProjectionMode=%q, want %q from new primary", entry.EngineProjectionMode, "degraded")
 	}
 }
+
+func TestRegistry_StaleCleanup_SkipsRecentlyRegisteredEntry(t *testing.T) {
+	r := NewBlockVolumeRegistry()
+	r.MarkBlockCapable("vs1:8080")
+
+	// Register a volume — RegisteredAt is set automatically.
+	if err := r.Register(&BlockVolumeEntry{
+		Name:         "vol-grace",
+		VolumeServer: "vs1:8080",
+		Path:         "/blocks/vol-grace.blk",
+		Status:       StatusActive,
+		Role:         blockvol.RoleToWire(blockvol.RolePrimary),
+	}); err != nil {
+		t.Fatalf("register: %v", err)
+	}
+
+	// Authoritative heartbeat from vs1 that does NOT report this volume.
+	// Without grace period, this would delete the entry.
+	r.UpdateFullHeartbeatWithInventoryAuthority("vs1:8080", nil, "", true)
+
+	// Entry should survive — it was just registered.
+	entry, ok := r.Lookup("vol-grace")
+	if !ok {
+		t.Fatal("recently registered entry was deleted by stale cleanup — grace period not working")
+	}
+	if entry.Name != "vol-grace" {
+		t.Fatalf("entry name=%q, want vol-grace", entry.Name)
+	}
+}
+
+func TestRegistry_StaleCleanup_DeletesOldUnreportedEntry(t *testing.T) {
+	r := NewBlockVolumeRegistry()
+	r.MarkBlockCapable("vs1:8080")
+
+	// Register a volume with RegisteredAt in the past (beyond grace period).
+	if err := r.Register(&BlockVolumeEntry{
+		Name:         "vol-stale",
+		VolumeServer: "vs1:8080",
+		Path:         "/blocks/vol-stale.blk",
+		Status:       StatusActive,
+		Role:         blockvol.RoleToWire(blockvol.RolePrimary),
+		RegisteredAt: time.Now().Add(-60 * time.Second), // 60s ago, past grace
+	}); err != nil {
+		t.Fatalf("register: %v", err)
+	}
+
+	// Authoritative heartbeat without this volume.
+	r.UpdateFullHeartbeatWithInventoryAuthority("vs1:8080", nil, "", true)
+
+	// Entry should be deleted — it's old and not reported.
+	_, ok := r.Lookup("vol-stale")
+	if ok {
+		t.Fatal("old unreported entry survived stale cleanup — grace period should not protect it")
+	}
+}

weed/server/volume_grpc_client_to_master.go

@@ -349,7 +349,10 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp
 			return
 		case <-vs.stopChan:
 			var volumeMessages []*master_pb.VolumeInformationMessage
-			blockInventoryAuthoritative := true
+			// Shutdown beat: clear regular volumes but do NOT claim block
+			// inventory authority. The block registry entry must survive
+			// shutdown so failoverBlockVolumes can promote the replica.
+			noBlockAuthority := false
 			emptyBeat := &master_pb.Heartbeat{
 				Ip:                                ip,
 				Port:                              port,
@@ -359,8 +362,8 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp
 				Rack:                              rack,
 				Volumes:                           volumeMessages,
 				HasNoVolumes:                      len(volumeMessages) == 0,
-				HasNoBlockVolumes:                 vs.blockService != nil,
-				BlockVolumeInventoryAuthoritative: &blockInventoryAuthoritative,
+				HasNoBlockVolumes:                 false,
+				BlockVolumeInventoryAuthoritative: &noBlockAuthority,
 			}
 			glog.V(1).Infof("volume server %s:%d stops and deletes all volumes", vs.store.Ip, vs.store.Port)
 			if err = stream.Send(emptyBeat); err != nil {