From 34bab4bc5816a13a8e287fc278c30a61b6964b9f Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 22 Jul 2010 07:43:17 +0000 Subject: [PATCH] Micro-optimized the IB completion handlers for modes thread=0 and thread=2. Fixed another session unregistration race. git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@1857 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- srpt/src/ib_srpt.c | 98 ++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/srpt/src/ib_srpt.c b/srpt/src/ib_srpt.c index b016dfe7c..d3372f808 100644 --- a/srpt/src/ib_srpt.c +++ b/srpt/src/ib_srpt.c @@ -1752,31 +1752,38 @@ static void srpt_rcv_completion(struct ib_cq *cq, void *ctx) EXTRACHECKS_WARN_ON(thread == MODE_SINGLE_THREADED); + if (atomic_read(&ch->state) == RDMA_CHANNEL_DISCONNECTING) + return; + spin_lock_irqsave(&ch->recv_lock, flags); - ib_req_notify_cq(ch->rcq, IB_CQ_NEXT_COMP); - while (ib_poll_cq(ch->rcq, 1, &wc) > 0) { - int req_lim; + do { + while (ib_poll_cq(ch->rcq, 1, &wc) > 0) { + int req_lim; - if (unlikely(wc.status)) { - spin_unlock_irqrestore(&ch->recv_lock, flags); - PRINT_INFO("receiving wr_id %u failed with status %d", - (unsigned)(wc.wr_id & ~SRPT_OP_RECV), - wc.status); - spin_lock_irqsave(&ch->recv_lock, flags); - continue; - } + if (unlikely(wc.status)) { + spin_unlock_irqrestore(&ch->recv_lock, flags); + PRINT_INFO("receiving wr_id %u failed with" + " status %d", + (unsigned)(wc.wr_id & ~SRPT_OP_RECV), + wc.status); + spin_lock_irqsave(&ch->recv_lock, flags); + continue; + } - req_lim = atomic_dec_return(&ch->req_lim); - if (unlikely(req_lim < 0)) - PRINT_ERROR("req_lim = %d < 0", req_lim); - ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV]; - srpt_handle_new_iu(ch, ioctx); + req_lim = atomic_dec_return(&ch->req_lim); + if (unlikely(req_lim < 0)) + PRINT_ERROR("req_lim = %d < 0", req_lim); + ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV]; + srpt_handle_new_iu(ch, ioctx); #if defined(CONFIG_SCST_DEBUG) - if (interrupt_processing_delay_in_us <= MAX_UDELAY_MS * 1000) - udelay(interrupt_processing_delay_in_us); + if (interrupt_processing_delay_in_us + <= MAX_UDELAY_MS * 1000) + udelay(interrupt_processing_delay_in_us); #endif - } + } + } while (ib_req_notify_cq(ch->rcq, IB_CQ_NEXT_COMP + | IB_CQ_REPORT_MISSED_EVENTS) > 0); spin_unlock_irqrestore(&ch->recv_lock, flags); } @@ -1793,35 +1800,39 @@ static void srpt_send_completion(struct ib_cq *cq, void *ctx) EXTRACHECKS_WARN_ON(thread == MODE_SINGLE_THREADED); - ib_req_notify_cq(ch->scq, IB_CQ_NEXT_COMP); - while (ib_poll_cq(ch->scq, 1, &wc) > 0) { - if (unlikely(wc.status)) { - PRINT_INFO("sending response for wr_id %u failed with" - " status %d", - (unsigned)(wc.wr_id & ~SRPT_OP_RECV), - wc.status); - srpt_handle_send_err_comp(ch, wc.wr_id, context); - continue; - } + do { + while (ib_poll_cq(ch->scq, 1, &wc) > 0) { + if (unlikely(wc.status)) { + PRINT_INFO("sending response for wr_id %u" + " failed with status %d", + (unsigned)(wc.wr_id & ~SRPT_OP_RECV), + wc.status); + srpt_handle_send_err_comp(ch, wc.wr_id, + context); + continue; + } - ioctx = sdev->ioctx_ring[wc.wr_id]; - atomic_add(wc.opcode == IB_WC_SEND ? 1 : ioctx->n_rdma, - &ch->sq_wr_avail); - if (wc.opcode == IB_WC_SEND) - srpt_handle_send_comp(ch, ioctx, context); - else { + ioctx = sdev->ioctx_ring[wc.wr_id]; + atomic_add(wc.opcode == IB_WC_SEND ? 1 : ioctx->n_rdma, + &ch->sq_wr_avail); + if (wc.opcode == IB_WC_SEND) + srpt_handle_send_comp(ch, ioctx, context); + else { #if defined(CONFIG_SCST_DEBUG) - WARN_ON(wc.opcode != IB_WC_RDMA_READ); - WARN_ON(ioctx->n_rdma <= 0); + WARN_ON(wc.opcode != IB_WC_RDMA_READ); + WARN_ON(ioctx->n_rdma <= 0); #endif - srpt_handle_rdma_comp(ch, ioctx, context); - } + srpt_handle_rdma_comp(ch, ioctx, context); + } #if defined(CONFIG_SCST_DEBUG) - if (interrupt_processing_delay_in_us <= MAX_UDELAY_MS * 1000) - udelay(interrupt_processing_delay_in_us); + if (interrupt_processing_delay_in_us + <= MAX_UDELAY_MS * 1000) + udelay(interrupt_processing_delay_in_us); #endif - } + } + } while (ib_req_notify_cq(ch->scq, IB_CQ_NEXT_COMP + | IB_CQ_REPORT_MISSED_EVENTS) > 0); } /** @@ -1837,6 +1848,9 @@ static void srpt_rcv_completion_st(struct ib_cq *cq, void *ctx) EXTRACHECKS_WARN_ON(thread != MODE_SINGLE_THREADED); + if (atomic_read(&ch->state) == RDMA_CHANNEL_DISCONNECTING) + return; + spin_lock_irqsave(&ch->recv_lock, flags); ib_req_notify_cq(ch->rcq, IB_CQ_NEXT_COMP); while (ib_poll_cq(ch->rcq, 1, &wc) > 0) {