diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index d6db78a4b..0f037dcd1 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -106,10 +106,11 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest vl.SetLastGrowCount(req.WritableVolumeCount) var ( - lastErr error - maxTimeout = time.Second * 10 - startTime = time.Now() - initiatedGrow bool + lastErr error + maxTimeout = time.Second * 10 + startTime = time.Now() + initiatedGrow bool + repickedAfterGrow bool ) for time.Now().Sub(startTime) < maxTimeout { @@ -145,6 +146,14 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest // Unavailable: clients retry it (assign_file_id.go) without // invalidating the shared master connection. if initiatedGrow != vl.HasGrowRequest() { + // The failed pick above may predate the growth concluding, so + // re-pick once before shedding: growth registers its volumes + // before clearing the flag, and shedding here would bounce the + // client off a volume that just landed. + if initiatedGrow && !repickedAfterGrow { + repickedAfterGrow = true + continue + } return nil, status.Errorf(codes.ResourceExhausted, "no writable volumes for %s, volume growth in progress", option.String()) } } diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index 0a8d30b34..f028a5a04 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -158,10 +158,11 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) var ( - lastErr error - maxTimeout = time.Second * 10 - startTime = time.Now() - initiatedGrow bool + lastErr error + maxTimeout = time.Second * 10 + startTime = time.Now() + initiatedGrow bool + repickedAfterGrow bool ) if !ms.Topo.DataCenterExists(option.DataCenter) { @@ -195,6 +196,12 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) // See Assign: only the initiator waits, and only while the // growth it triggered is still pending. if initiatedGrow != vl.HasGrowRequest() { + // See Assign: re-pick once after the growth concludes before + // shedding — the failed pick may predate the conclusion. + if initiatedGrow && !repickedAfterGrow { + repickedAfterGrow = true + continue + } w.Header().Set("Retry-After", "1") writeJsonQuiet(w, r, http.StatusServiceUnavailable, operation.AssignResult{ Error: fmt.Sprintf("no writable volumes for %s, volume growth in progress", option.String()),