From 543989b3a2eef98ab74b56682c6d198cd1c735e0 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 19 May 2020 19:21:26 +0000 Subject: [PATCH] ib_srpt: Prevent that large RDMA transfers trigger QP termination Running fio at the initiator side with buffer size 1 MiB and when using direct I/O triggers the following error with a ConnectX-3 and Linux kernel v4.19 or later or CentOS 7.7 or later at the target side: ib_srpt: RDMA t 4 for idx 20 failed with status 5. If this has not been triggered by a cable pull, please check the involved IB HCA's and cables. This was observed with firmware version 2.42.5000 but this is probably independent of the firmware version. Fix this by reducing the send SGE QP limit. This patch increases ch->max_recv_sge for older kernel versions. Reported-by: Chesnokov Gleb git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@8938 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- srpt/src/ib_srpt.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/srpt/src/ib_srpt.c b/srpt/src/ib_srpt.c index 165c1a432..51286248c 100644 --- a/srpt/src/ib_srpt.c +++ b/srpt/src/ib_srpt.c @@ -2329,21 +2329,22 @@ retry: qp_init->sq_sig_type = IB_SIGNAL_REQ_WR; qp_init->qp_type = IB_QPT_RC; qp_init->cap.max_send_wr = sq_size; +#if HAVE_DEV_ATTR_MAX_SEND_SGE + ch->max_send_sge = sdev->dev_attr.max_send_sge; + ch->max_recv_sge = sdev->dev_attr.max_recv_sge; +#else + ch->max_send_sge = sdev->dev_attr.max_sge; + ch->max_recv_sge = ch->max_send_sge; +#endif /* * For max_sge values > 2 * max_sge_delta, subtract max_sge_delta. For * max_sge values < max_sge_delta, use max_sge. For intermediate * max_sge values, use max_sge_delta. */ -#if HAVE_DEV_ATTR_MAX_SEND_SGE - ch->max_send_sge = sdev->dev_attr.max_send_sge; - ch->max_recv_sge = sdev->dev_attr.max_recv_sge; -#else - ch->max_send_sge = sdev->dev_attr.max_sge - + ch->max_send_sge -= min_t(unsigned, max_sge_delta, max_t(int, 0, - sdev->dev_attr.max_sge - max_sge_delta)); - ch->max_recv_sge = ch->max_send_sge; -#endif + ch->max_send_sge - max_sge_delta)); qp_init->cap.max_send_sge = ch->max_send_sge; qp_init->cap.max_recv_sge = ch->max_recv_sge; if (sdev->use_srq) {