master: assign re-picks once after its growth concludes instead of shedding (#10348)

The initiator's shed check (initiatedGrow != HasGrowRequest) compares
against an err from a PickForWrite that may predate the growth
concluding: the grower registers its volumes before clearing the flag,
so when growth lands between the failed pick and the check, the assign
shed ResourceExhausted even though a writable volume was already
registered. Re-pick once after observing the conclusion and shed only
if the volume layout still has nothing writable. Applies to both the
gRPC Assign and the HTTP dirAssign paths, which share the shed logic.

Flaked in CI as TestAssignInitiatorWaitsForItsOwnGrowth; reproduced
deterministically by widening the enqueue-to-check window.

weed/server/master_grpc_server_assign.go

@@ -106,10 +106,11 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 	vl.SetLastGrowCount(req.WritableVolumeCount)
 
 	var (
-		lastErr       error
-		maxTimeout    = time.Second * 10
-		startTime     = time.Now()
-		initiatedGrow bool
+		lastErr           error
+		maxTimeout        = time.Second * 10
+		startTime         = time.Now()
+		initiatedGrow     bool
+		repickedAfterGrow bool
 	)
 
 	for time.Now().Sub(startTime) < maxTimeout {
@@ -145,6 +146,14 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 				// Unavailable: clients retry it (assign_file_id.go) without
 				// invalidating the shared master connection.
 				if initiatedGrow != vl.HasGrowRequest() {
+					// The failed pick above may predate the growth concluding, so
+					// re-pick once before shedding: growth registers its volumes
+					// before clearing the flag, and shedding here would bounce the
+					// client off a volume that just landed.
+					if initiatedGrow && !repickedAfterGrow {
+						repickedAfterGrow = true
+						continue
+					}
 					return nil, status.Errorf(codes.ResourceExhausted, "no writable volumes for %s, volume growth in progress", option.String())
 				}
 			}

weed/server/master_server_handlers.go

@@ -158,10 +158,11 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
 	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType)
 
 	var (
-		lastErr       error
-		maxTimeout    = time.Second * 10
-		startTime     = time.Now()
-		initiatedGrow bool
+		lastErr           error
+		maxTimeout        = time.Second * 10
+		startTime         = time.Now()
+		initiatedGrow     bool
+		repickedAfterGrow bool
 	)
 
 	if !ms.Topo.DataCenterExists(option.DataCenter) {
@@ -195,6 +196,12 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
 				// See Assign: only the initiator waits, and only while the
 				// growth it triggered is still pending.
 				if initiatedGrow != vl.HasGrowRequest() {
+					// See Assign: re-pick once after the growth concludes before
+					// shedding — the failed pick may predate the conclusion.
+					if initiatedGrow && !repickedAfterGrow {
+						repickedAfterGrow = true
+						continue
+					}
 					w.Header().Set("Retry-After", "1")
 					writeJsonQuiet(w, r, http.StatusServiceUnavailable, operation.AssignResult{
 						Error: fmt.Sprintf("no writable volumes for %s, volume growth in progress", option.String()),