mirrors/scst
1
0
Fork 0
mirror of https://github.com/SCST-project/scst.git synced 2026-05-17 10:41:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

ib_srpt: Defer destroying the QP until the TimeWait state has been left

Browse Source 
This is necessary to avoid that a login gets rejected due to reusing a
queue pair number that has not yet been freed by the target side.


git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@5356 d57e44dd-8a1f-0410-8b47-8ef2f437770f
This commit is contained in:
Bart Van Assche
2014-03-20 15:24:18 +00:00
parent 7efb024a7d
commit 46aa72be44
2 changed files with 23 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
srpt/src/ib_srpt.c
View File

@@ -181,7 +181,6 @@ static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
#endif /*CONFIG_SCST_PROC*/
static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
struct srpt_send_ioctx *ioctx);
static void srpt_drain_channel(struct ib_cm_id *cm_id);
static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch);
static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
@@ -329,8 +328,6 @@ static const char *get_ch_state_name(enum rdma_ch_state s)
return "live";
case CH_DISCONNECTING:
return "disconnecting";
case CH_DRAINING:
return "draining";
}
return "???";
}
@@ -2036,19 +2033,6 @@ static void srpt_unreg_sess(struct scst_session *scst_sess)
sdev, ch->rq_size,
ch->max_rsp_size, DMA_TO_DEVICE);
/*
* Note: if a DREQ is received after ch->dreq_received has been read,
* ib_destroy_cm_id() will send a DREP.
*
*/
if (ch->dreq_received) {
if (ib_send_cm_drep(ch->cm_id, NULL, 0) >= 0)
PRINT_INFO("Received DREQ and sent DREP for session %s",
ch->sess_name);
else
PRINT_ERROR("Sending DREP failed");
}
/*
* If the connection is still established, ib_destroy_cm_id() will
* send a DREQ.
@@ -2251,8 +2235,9 @@ static bool __srpt_close_ch(struct srpt_rdma_ch *ch)
ret = srpt_ch_qp_err(ch);
if (ret < 0)
PRINT_ERROR("Setting queue pair in error state"
" failed: %d", ret);
PRINT_ERROR("%s: changing queue pair into error state"
" failed: %d", ch->sess_name, ret);
kref_put(&ch->kref, srpt_free_ch);
spin_lock_irq(&srpt_tgt->spinlock);
@@ -2273,34 +2258,6 @@ static void srpt_close_ch(struct srpt_rdma_ch *ch)
spin_unlock_irq(&srpt_tgt->spinlock);
}
/**
* srpt_drain_channel() - Drain a channel by resetting the IB queue pair.
* @cm_id: Pointer to the CM ID of the channel to be drained.
*
* Note: Must be called from inside srpt_cm_handler to avoid a race between
* accessing sdev->spinlock and the call to kfree(sdev) in srpt_remove_one()
* (the caller of srpt_cm_handler holds the cm_id spinlock; srpt_remove_one()
* waits until all target sessions for the associated IB device have been
* unregistered and target session registration involves a call to
* ib_destroy_cm_id(), which locks the cm_id spinlock and hence waits until
* this function has finished).
*/
static void srpt_drain_channel(struct ib_cm_id *cm_id)
{
struct srpt_rdma_ch *ch;
int ret;
WARN_ON_ONCE(irqs_disabled());
ch = cm_id->context;
if (srpt_set_ch_state(ch, CH_DRAINING)) {
ret = srpt_ch_qp_err(ch);
if (ret < 0)
PRINT_ERROR("Setting queue pair in error state"
" failed: %d", ret);
}
}
static void __srpt_close_all_ch(struct srpt_tgt *srpt_tgt)
{
struct srpt_nexus *nexus;
@@ -2313,12 +2270,11 @@ static void __srpt_close_all_ch(struct srpt_tgt *srpt_tgt)
restart:
list_for_each_entry(nexus, &srpt_tgt->nexus_list, entry) {
list_for_each_entry(ch, &nexus->ch_list, list) {
if (ch->state >= CH_DISCONNECTING)
if (ib_send_cm_dreq(ch->cm_id, NULL, 0) < 0)
continue;
PRINT_INFO("Closing channel %s because target %s has"
" been disabled", ch->sess_name,
srpt_tgt->scst_tgt->tgt_name);
WARN_ON_ONCE(!__srpt_close_ch(ch));
goto restart;
}
}
@@ -2656,7 +2612,7 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
restart:
list_for_each_entry(ch2, &nexus->ch_list, list) {
if (!__srpt_close_ch(ch2))
if (ib_send_cm_dreq(ch2->cm_id, NULL, 0) < 0)
continue;
PRINT_INFO("Relogin - closed existing channel %s",
ch2->sess_name);
@@ -2784,7 +2740,6 @@ out:
static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
{
PRINT_INFO("Received InfiniBand REJ packet for cm_id %p.", cm_id);
srpt_drain_channel(cm_id);
}
/**
@@ -2816,25 +2771,32 @@ static void srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
{
struct srpt_rdma_ch *ch = cm_id->context;
PRINT_INFO("Received InfiniBand TimeWait exit for cm_id %p.", cm_id);
srpt_drain_channel(cm_id);
srpt_close_ch(ch);
}
static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
{
PRINT_INFO("Received InfiniBand REP error for cm_id %p.", cm_id);
srpt_drain_channel(cm_id);
}
/**
* srpt_cm_dreq_recv() - Process reception of a DREQ message.
*/
static void srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
{
struct srpt_rdma_ch *ch = cm_id->context;
int ret;
ch->dreq_received = true;
srpt_set_ch_state(ch, CH_DISCONNECTING);
ret = ib_send_cm_drep(cm_id, NULL, 0);
if (ret < 0)
PRINT_ERROR("%s: sending DREP failed", ch->sess_name);
srpt_close_ch(ch);
return ret;
}
/**
@@ -2842,8 +2804,10 @@ static void srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
*/
static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
{
struct srpt_rdma_ch *ch = cm_id->context;
PRINT_INFO("Received InfiniBand DREP message for cm_id %p.", cm_id);
srpt_drain_channel(cm_id);
srpt_close_ch(ch);
}
/**
@@ -2876,7 +2840,7 @@ static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
srpt_cm_rtu_recv(cm_id);
break;
case IB_CM_DREQ_RECEIVED:
srpt_cm_dreq_recv(cm_id);
ret = srpt_cm_dreq_recv(cm_id);
break;
case IB_CM_DREP_RECEIVED:
srpt_cm_drep_recv(cm_id);
@@ -3204,7 +3168,7 @@ static int srpt_perform_rdmas(struct srpt_rdma_ch *ch,
}
PRINT_INFO("Waiting until RDMA abort finished [%d]",
ioctx->ioctx.index);
while (ch->state != CH_DRAINING && !ioctx->rdma_aborted) {
while (ch->state == CH_LIVE && !ioctx->rdma_aborted) {
PRINT_INFO("Waiting until RDMA abort finished [%d]",
ioctx->ioctx.index);
msleep(1000);
@@ -3543,7 +3507,7 @@ static int srpt_close_session(struct scst_session *sess)
{
struct srpt_rdma_ch *ch = scst_sess_get_tgt_priv(sess);
srpt_close_ch(ch);
ib_send_cm_dreq(ch->cm_id, NULL, 0);
return 0;
}

4
srpt/src/ib_srpt.h
View File

@@ -282,13 +282,11 @@ struct srpt_send_ioctx {
* @CH_DISCONNECTING: DREQ has been received and waiting for DREP or DREQ has
* been sent and waiting for DREP or channel is being closed
* for another reason.
* @CH_DRAINING: QP is in ERR state.
*/
enum rdma_ch_state {
CH_CONNECTING,
CH_LIVE,
CH_DISCONNECTING,
CH_DRAINING,
};
/**
@@ -319,7 +317,6 @@ enum rdma_ch_state {
* @wc: Work completion array.
* @state: channel state. See also enum rdma_ch_state.
* @processing_wait_list: Whether or not cmd_wait_list is being processed.
* @dreq_received: Whether an IB CM DREQ event has been received.
* @last_wqe_received: Whether the Last WQE QP event has been received.
* @list: Entry in srpt_nexus.ch_list;
* @cmd_wait_list: list of SCST commands that arrived before the RTU event. This
@@ -354,7 +351,6 @@ struct srpt_rdma_ch {
struct list_head cmd_wait_list;
uint16_t pkey_index;
bool processing_wait_list;
bool dreq_received;
bool last_wqe_received;
struct scst_session *scst_sess;