From 8681a0124a7c0985b55d1d4facff8f2622c612a1 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Sun, 26 Jun 2011 14:46:23 +0000 Subject: [PATCH] ib_srpt: Avoid that failed multipart RDMA transfers cause havoc. Multipart RDMA transfers can fail after one or more but not all RDMA transfers have been initiated because either an IB cable has been pulled or the ib_srpt kernel module has been unloaded while an RDMA transfer is being set up. git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@3632 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- srpt/src/ib_srpt.c | 93 ++++++++++++++++++++++++++++++++-------------- srpt/src/ib_srpt.h | 16 +++++++- 2 files changed, 79 insertions(+), 30 deletions(-) diff --git a/srpt/src/ib_srpt.c b/srpt/src/ib_srpt.c index fcd533cdb..2e84a9787 100644 --- a/srpt/src/ib_srpt.c +++ b/srpt/src/ib_srpt.c @@ -890,7 +890,7 @@ static int srpt_post_recv(struct srpt_device *sdev, struct ib_recv_wr wr, *bad_wr; BUG_ON(!sdev); - wr.wr_id = encode_wr_id(IB_WC_RECV, ioctx->ioctx.index); + wr.wr_id = encode_wr_id(SRPT_RECV, ioctx->ioctx.index); list.addr = ioctx->ioctx.dma; list.length = srp_max_req_size; @@ -943,7 +943,7 @@ static int srpt_post_send(struct srpt_rdma_ch *ch, list.lkey = sdev->mr->lkey; wr.next = NULL; - wr.wr_id = encode_wr_id(IB_WC_SEND, ioctx->ioctx.index); + wr.wr_id = encode_wr_id(SRPT_SEND, ioctx->ioctx.index); wr.sg_list = &list; wr.num_sge = 1; wr.opcode = IB_WR_SEND; @@ -1409,6 +1409,7 @@ static void srpt_handle_send_comp(struct srpt_rdma_ch *ch, */ static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch, struct srpt_send_ioctx *ioctx, + enum srpt_opcode opcode, enum scst_exec_context context) { struct scst_cmd *scmnd; @@ -1417,7 +1418,7 @@ static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch, srpt_adjust_srq_wr_avail(ch, ioctx->n_rdma); scmnd = ioctx->scmnd; - if (scmnd) { + if (opcode == SRPT_RDMA_READ_LAST && scmnd) { if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA, SRPT_STATE_DATA_IN)) scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS, @@ -1425,8 +1426,13 @@ static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch, else PRINT_ERROR("%s[%d]: wrong state = %d", __func__, __LINE__, ioctx->state); - } else - PRINT_ERROR("%s[%d]: scmnd == NULL", __func__, __LINE__); + } else if (opcode == SRPT_RDMA_ABORT) { + ioctx->rdma_aborted = true; + } else { + WARN_ON(opcode != SRPT_RDMA_READ_LAST); + PRINT_ERROR("%s[%d]: scmnd == NULL (opcode %d)", __func__, + __LINE__, opcode); + } } /** @@ -1434,7 +1440,7 @@ static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch, */ static void srpt_handle_rdma_err_comp(struct srpt_rdma_ch *ch, struct srpt_send_ioctx *ioctx, - u8 opcode, + enum srpt_opcode opcode, enum scst_exec_context context) { struct scst_cmd *scmnd; @@ -1444,7 +1450,7 @@ static void srpt_handle_rdma_err_comp(struct srpt_rdma_ch *ch, state = ioctx->state; if (scmnd) { switch (opcode) { - case IB_WC_RDMA_READ: + case SRPT_RDMA_READ_LAST: if (ioctx->n_rdma <= 0) { PRINT_ERROR("Received invalid RDMA read error" " completion with idx %d", @@ -1458,7 +1464,7 @@ static void srpt_handle_rdma_err_comp(struct srpt_rdma_ch *ch, PRINT_ERROR("%s[%d]: wrong state = %d", __func__, __LINE__, state); break; - case IB_WC_RDMA_WRITE: + case SRPT_RDMA_WRITE_LAST: scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED); break; @@ -1861,34 +1867,33 @@ static void srpt_process_send_completion(struct ib_cq *cq, { struct srpt_send_ioctx *send_ioctx; uint32_t index; - u8 opcode; + enum srpt_opcode opcode; index = idx_from_wr_id(wc->wr_id); opcode = opcode_from_wr_id(wc->wr_id); send_ioctx = ch->ioctx_ring[index]; if (wc->status == IB_WC_SUCCESS) { - if (opcode == IB_WC_SEND) + if (opcode == SRPT_SEND) srpt_handle_send_comp(ch, send_ioctx, context); else { - EXTRACHECKS_WARN_ON(wc->opcode != IB_WC_RDMA_READ); - srpt_handle_rdma_comp(ch, send_ioctx, context); + EXTRACHECKS_WARN_ON(opcode != SRPT_RDMA_ABORT && + wc->opcode != IB_WC_RDMA_READ); + srpt_handle_rdma_comp(ch, send_ioctx, opcode, context); } } else { - if (opcode == IB_WC_SEND) { + if (opcode == SRPT_SEND) { PRINT_INFO("sending response for idx %u failed with" " status %d", index, wc->status); srpt_handle_send_err_comp(ch, wc->wr_id, context); - } else { - PRINT_INFO("RDMA %s for idx %u failed with status %d", - opcode == IB_WC_RDMA_READ ? "read" - : opcode == IB_WC_RDMA_WRITE ? "write" - : "???", index, wc->status); + } else if (opcode != SRPT_RDMA_MID) { + PRINT_INFO("RDMA t %d for idx %u failed with status %d", + opcode, index, wc->status); srpt_handle_rdma_err_comp(ch, send_ioctx, opcode, context); } } - while (unlikely(opcode == IB_WC_SEND + while (unlikely(opcode == SRPT_SEND && !list_empty(&ch->cmd_wait_list) && ch->state == CH_LIVE && (send_ioctx = srpt_get_send_ioctx(ch)) != NULL)) { @@ -1918,7 +1923,7 @@ static bool srpt_process_completion(struct ib_cq *cq, ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); while ((n = ib_poll_cq(cq, ARRAY_SIZE(ch->wc), wc)) > 0) { for (i = 0; i < n; i++) { - if (opcode_from_wr_id(wc[i].wr_id) & IB_WC_RECV) + if (opcode_from_wr_id(wc[i].wr_id) == SRPT_RECV) srpt_process_rcv_completion(cq, ch, rcv_context, &wc[i]); else @@ -1964,6 +1969,11 @@ static int srpt_compl_thread(void *arg) WARN_ON(ch->state != CH_RELEASING); scst_unregister_session(ch->scst_sess, false, srpt_free_ch); + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + } + return 0; } @@ -2912,29 +2922,35 @@ static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, int i; int ret; int sq_wr_avail; + const int n_rdma = ioctx->n_rdma; if (dir == SCST_DATA_WRITE) { ret = -ENOMEM; - sq_wr_avail = srpt_adjust_srq_wr_avail(ch, -ioctx->n_rdma); + sq_wr_avail = srpt_adjust_srq_wr_avail(ch, -n_rdma); if (sq_wr_avail < 0) { PRINT_WARNING("IB send queue full (needed %d)", - ioctx->n_rdma); + n_rdma); goto out; } } + ioctx->rdma_aborted = false; ret = 0; riu = ioctx->rdma_ius; memset(&wr, 0, sizeof wr); - for (i = 0; i < ioctx->n_rdma; ++i, ++riu) { + for (i = 0; i < n_rdma; ++i, ++riu) { if (dir == SCST_DATA_READ) { wr.opcode = IB_WR_RDMA_WRITE; - wr.wr_id = encode_wr_id(IB_WC_RDMA_WRITE, + wr.wr_id = encode_wr_id(i == n_rdma - 1 ? + SRPT_RDMA_WRITE_LAST : + SRPT_RDMA_MID, ioctx->ioctx.index); } else { wr.opcode = IB_WR_RDMA_READ; - wr.wr_id = encode_wr_id(IB_WC_RDMA_READ, + wr.wr_id = encode_wr_id(i == n_rdma - 1 ? + SRPT_RDMA_READ_LAST : + SRPT_RDMA_MID, ioctx->ioctx.index); } wr.next = NULL; @@ -2944,17 +2960,38 @@ static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, wr.sg_list = riu->sge; /* only get completion event for the last rdma wr */ - if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE) + if (i == (n_rdma - 1) && dir == SCST_DATA_WRITE) wr.send_flags = IB_SEND_SIGNALED; ret = ib_post_send(ch->qp, &wr, &bad_wr); if (ret) - goto out; + break; + } + + if (ret) + PRINT_ERROR("%s[%d]: ib_post_send() returned %d for %d/%d", + __func__, __LINE__, ret, i, n_rdma); + if (ret && i > 0) { + wr.num_sge = 0; + wr.wr_id = encode_wr_id(SRPT_RDMA_ABORT, ioctx->ioctx.index); + wr.send_flags = IB_SEND_SIGNALED; + while (ch->state == CH_LIVE && + ib_post_send(ch->qp, &wr, &bad_wr) != 0) { + PRINT_INFO("Trying to abort failed RDMA transfer [%d]", + ioctx->ioctx.index); + msleep(1000); + } + while (ch->state != CH_RELEASING && !ioctx->rdma_aborted) { + PRINT_INFO("Waiting until RDMA abort finished [%d]", + ioctx->ioctx.index); + msleep(1000); + } + PRINT_INFO("%s[%d]: done", __func__, __LINE__); } out: if (unlikely(dir == SCST_DATA_WRITE && ret < 0)) - srpt_adjust_srq_wr_avail(ch, ioctx->n_rdma); + srpt_adjust_srq_wr_avail(ch, n_rdma); return ret; } diff --git a/srpt/src/ib_srpt.h b/srpt/src/ib_srpt.h index 5774120ba..9e9f28721 100644 --- a/srpt/src/ib_srpt.h +++ b/srpt/src/ib_srpt.h @@ -128,12 +128,21 @@ enum { DEFAULT_MAX_RDMA_SIZE = 65536, }; -static inline u64 encode_wr_id(u8 opcode, u32 idx) +enum srpt_opcode { + SRPT_RECV, + SRPT_SEND, + SRPT_RDMA_MID, + SRPT_RDMA_ABORT, + SRPT_RDMA_READ_LAST, + SRPT_RDMA_WRITE_LAST, +}; + +static inline u64 encode_wr_id(enum srpt_opcode opcode, u32 idx) { return ((u64)opcode << 32) | idx; } -static inline u8 opcode_from_wr_id(u64 wr_id) +static inline enum srpt_opcode opcode_from_wr_id(u64 wr_id) { return wr_id >> 32; } @@ -215,6 +224,8 @@ struct srpt_tsk_mgmt { * @sg: Pointer to sg-list associated with this I/O context. * @spinlock: Protects 'state'. * @state: I/O context state. + * @rdma_aborted: If initiating a multipart RDMA transfer failed, whether + * the already initiated transfers have finished. * @scmnd: SCST command data structure. * @dir: * @free_list: Node in srpt_rdma_ch.free_list. @@ -237,6 +248,7 @@ struct srpt_send_ioctx { struct list_head free_list; spinlock_t spinlock; enum srpt_command_state state; + bool rdma_aborted; struct scst_cmd *scmnd; scst_data_direction dir; int sg_cnt;