From 1fda7aa7f13fcc8234e3334e8f9f31dd4e588900 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 16 Jul 2026 13:55:38 -0700 Subject: [PATCH] master: assign re-picks once after its growth concludes instead of shedding (#10348) The initiator's shed check (initiatedGrow != HasGrowRequest) compares against an err from a PickForWrite that may predate the growth concluding: the grower registers its volumes before clearing the flag, so when growth lands between the failed pick and the check, the assign shed ResourceExhausted even though a writable volume was already registered. Re-pick once after observing the conclusion and shed only if the volume layout still has nothing writable. Applies to both the gRPC Assign and the HTTP dirAssign paths, which share the shed logic. Flaked in CI as TestAssignInitiatorWaitsForItsOwnGrowth; reproduced deterministically by widening the enqueue-to-check window. --- weed/server/master_grpc_server_assign.go | 17 +++++++++++++---- weed/server/master_server_handlers.go | 15 +++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index d6db78a4b..0f037dcd1 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -106,10 +106,11 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest vl.SetLastGrowCount(req.WritableVolumeCount) var ( - lastErr error - maxTimeout = time.Second * 10 - startTime = time.Now() - initiatedGrow bool + lastErr error + maxTimeout = time.Second * 10 + startTime = time.Now() + initiatedGrow bool + repickedAfterGrow bool ) for time.Now().Sub(startTime) < maxTimeout { @@ -145,6 +146,14 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest // Unavailable: clients retry it (assign_file_id.go) without // invalidating the shared master connection. if initiatedGrow != vl.HasGrowRequest() { + // The failed pick above may predate the growth concluding, so + // re-pick once before shedding: growth registers its volumes + // before clearing the flag, and shedding here would bounce the + // client off a volume that just landed. + if initiatedGrow && !repickedAfterGrow { + repickedAfterGrow = true + continue + } return nil, status.Errorf(codes.ResourceExhausted, "no writable volumes for %s, volume growth in progress", option.String()) } } diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index 0a8d30b34..f028a5a04 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -158,10 +158,11 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) var ( - lastErr error - maxTimeout = time.Second * 10 - startTime = time.Now() - initiatedGrow bool + lastErr error + maxTimeout = time.Second * 10 + startTime = time.Now() + initiatedGrow bool + repickedAfterGrow bool ) if !ms.Topo.DataCenterExists(option.DataCenter) { @@ -195,6 +196,12 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) // See Assign: only the initiator waits, and only while the // growth it triggered is still pending. if initiatedGrow != vl.HasGrowRequest() { + // See Assign: re-pick once after the growth concludes before + // shedding — the failed pick may predate the conclusion. + if initiatedGrow && !repickedAfterGrow { + repickedAfterGrow = true + continue + } w.Header().Set("Retry-After", "1") writeJsonQuiet(w, r, http.StatusServiceUnavailable, operation.AssignResult{ Error: fmt.Sprintf("no writable volumes for %s, volume growth in progress", option.String()),