mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-17 12:46:59 +00:00
Chris LuandGitHub
2386fa550a
A Canceled/DeadlineExceeded from the caller's per-request context was treated like a dead channel: it closed the shared cached ClientConn and cancelled every other in-flight RPC on it with "the client connection is closing". Under a burst of concurrent chunk assigns (e.g. a large S3 multipart upload) one slow assign hitting its 10s attempt timeout could poison the connection for all the rest, cascading into a flood of 500s. Thread the caller's context into shouldInvalidateConnection and only invalidate on Canceled/DeadlineExceeded while that context is still live, which isolates the genuine stale-channel signal (a peer restart behind a k8s Service VIP). To carry the context, add a ctx parameter to the existing WithGrpcClient, WithMasterClient, and WithMasterServerClient; the master assign and volume-lookup paths pass their per-attempt context and every other caller passes context.Background().
29 lines
1.2 KiB
Go
29 lines
1.2 KiB
Go
package operation
|
|
|
|
import (
|
|
"context"
|
|
|
|
"google.golang.org/grpc"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
|
)
|
|
|
|
func WithVolumeServerClient(streamingMode bool, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fn func(volume_server_pb.VolumeServerClient) error) error {
|
|
|
|
return pb.WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
|
|
client := volume_server_pb.NewVolumeServerClient(grpcConnection)
|
|
return fn(client)
|
|
}, volumeServer.ToGrpcAddress(), false, grpcDialOption)
|
|
|
|
}
|
|
|
|
// WithMasterServerClient threads the caller's per-request context into the
|
|
// connection-invalidation decision, so a Canceled/DeadlineExceeded from the
|
|
// caller's own timeout does not invalidate the shared cached master connection.
|
|
// Pass context.Background() when there is no per-request deadline to honor.
|
|
func WithMasterServerClient(ctx context.Context, streamingMode bool, masterServer pb.ServerAddress, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error {
|
|
return pb.WithMasterClient(ctx, streamingMode, masterServer, grpcDialOption, false, fn)
|
|
}
|