From 377c61ddd0fb97a02ecbc7225fbf7f6f6fbba707 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Sun, 21 Aug 2016 09:36:18 +0000 Subject: [PATCH] Merged revisions 6934-6958 from trunk ------------------------------------------------------------------------ r6934 | mlx-storage | 2016-08-17 10:17:37 +0300 (Wed, 17 Aug 2016) | 4 lines isert: make sure rdma_disconnect is called only once ------------------------------------------------------------------------ r6935 | mlx-storage | 2016-08-17 10:27:06 +0300 (Wed, 17 Aug 2016) | 8 lines isert: fix cleaning isert_con_dev ------------------------------------------------------------------------ r6936 | mlx-storage | 2016-08-17 10:27:11 +0300 (Wed, 17 Aug 2016) | 4 lines isert: add assertions for connection teardwon flow ------------------------------------------------------------------------ r6937 | mlx-storage | 2016-08-17 10:27:15 +0300 (Wed, 17 Aug 2016) | 8 lines isert: fix a possible extra refcount put of isert connection ------------------------------------------------------------------------ r6938 | mlx-storage | 2016-08-17 10:27:19 +0300 (Wed, 17 Aug 2016) | 4 lines isert: change wrong dev_conn bug on ------------------------------------------------------------------------ r6939 | mlx-storage | 2016-08-17 10:27:24 +0300 (Wed, 17 Aug 2016) | 6 lines isert: fix fops ioctl using invalid conn ------------------------------------------------------------------------ r6940 | mlx-storage | 2016-08-17 10:27:28 +0300 (Wed, 17 Aug 2016) | 9 lines isert: fix race between ioctl events and disconnect flow ------------------------------------------------------------------------ r6941 | mlx-storage | 2016-08-17 10:27:33 +0300 (Wed, 17 Aug 2016) | 7 lines isert: fix isert conn refcount release at unreachable event ------------------------------------------------------------------------ r6942 | mlx-storage | 2016-08-17 10:27:38 +0300 (Wed, 17 Aug 2016) | 7 lines isert: fix missing refcount cleanup at error flow ------------------------------------------------------------------------ r6943 | mlx-storage | 2016-08-17 10:27:43 +0300 (Wed, 17 Aug 2016) | 8 lines isert: add assertion if send/recv was posted after post drain ------------------------------------------------------------------------ r6944 | mlx-storage | 2016-08-17 10:27:48 +0300 (Wed, 17 Aug 2016) | 4 lines isert: replace kernel prints with scst macros ------------------------------------------------------------------------ r6945 | mlx-storage | 2016-08-17 10:27:53 +0300 (Wed, 17 Aug 2016) | 12 lines isert: fix a race between timewait exit handler and poll cq ------------------------------------------------------------------------ r6946 | mlx-storage | 2016-08-17 10:27:57 +0300 (Wed, 17 Aug 2016) | 9 lines isert: fix a race when drain wr is not the last flush ------------------------------------------------------------------------ r6947 | mlx-storage | 2016-08-17 10:28:02 +0300 (Wed, 17 Aug 2016) | 8 lines isert: fix isert connection kref leak at estabished event handler ------------------------------------------------------------------------ r6948 | mlx-storage | 2016-08-17 10:28:06 +0300 (Wed, 17 Aug 2016) | 7 lines isert: close isert connection earlier ------------------------------------------------------------------------ r6949 | mlx-storage | 2016-08-17 10:28:11 +0300 (Wed, 17 Aug 2016) | 4 lines isert: change dev conn_lock spinlock to mutex ------------------------------------------------------------------------ r6950 | mlx-storage | 2016-08-17 10:28:16 +0300 (Wed, 17 Aug 2016) | 13 lines isert: fix a race between calling to rdma_disconnect and connect flow ------------------------------------------------------------------------ r6951 | mlx-storage | 2016-08-17 10:28:20 +0300 (Wed, 17 Aug 2016) | 8 lines isert: fix isert connection kref leak ------------------------------------------------------------------------ r6952 | mlx-storage | 2016-08-17 10:28:25 +0300 (Wed, 17 Aug 2016) | 9 lines isert: faster release of isert_scst module ----------------------------------------------------------------------- r6953 | mlx-storage | 2016-08-17 10:28:29 +0300 (Wed, 17 Aug 2016) | 9 lines isert: fix races between conn fops read/write and disconnect flow ------------------------------------------------------------------------ r6954 | mlx-storage | 2016-08-17 10:28:34 +0300 (Wed, 17 Aug 2016) | 4 lines isert: add missing fd put on error flow ------------------------------------------------------------------------ r6955 | mlx-storage | 2016-08-17 10:28:38 +0300 (Wed, 17 Aug 2016) | 9 lines isert: fix working with freed conn object ------------------------------------------------------------------------ r6956 | mlx-storage | 2016-08-17 10:28:43 +0300 (Wed, 17 Aug 2016) | 8 lines isert: fix isert conn cleanup when rdma_accept fails ------------------------------------------------------------------------ r6957 | mlx-storage | 2016-08-17 10:28:47 +0300 (Wed, 17 Aug 2016) | 7 lines isert: add conn to portal conn list only if rdma_accept succeeded ------------------------------------------------------------------------ r6958 | mlx-storage | 2016-08-17 10:28:52 +0300 (Wed, 17 Aug 2016) | 4 lines isert: fix redundant module put on error flow when handling connect request ------------------------------------------------------------------------ git-svn-id: http://svn.code.sf.net/p/scst/svn/branches/3.2.x@6966 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- iscsi-scst/kernel/iscsi.c | 2 + iscsi-scst/kernel/isert-scst/iser.h | 14 +- iscsi-scst/kernel/isert-scst/iser_buf.c | 20 +- iscsi-scst/kernel/isert-scst/iser_datamover.c | 6 +- iscsi-scst/kernel/isert-scst/iser_datamover.h | 3 +- iscsi-scst/kernel/isert-scst/iser_global.c | 8 +- iscsi-scst/kernel/isert-scst/iser_pdu.c | 42 +- iscsi-scst/kernel/isert-scst/iser_rdma.c | 485 +++++++++++------- iscsi-scst/kernel/isert-scst/isert.c | 14 +- iscsi-scst/kernel/isert-scst/isert.h | 4 +- iscsi-scst/kernel/isert-scst/isert_login.c | 191 ++++--- 11 files changed, 487 insertions(+), 302 deletions(-) diff --git a/iscsi-scst/kernel/iscsi.c b/iscsi-scst/kernel/iscsi.c index 4dc07f0b4..58ba36dfd 100644 --- a/iscsi-scst/kernel/iscsi.c +++ b/iscsi-scst/kernel/iscsi.c @@ -2035,6 +2035,8 @@ static int scsi_cmnd_start(struct iscsi_cmnd *req) atomic_inc(&session->active_cmds); req->dec_active_cmds = 1; + sBUG_ON(session->scst_sess == NULL); + scst_cmd = scst_rx_cmd(session->scst_sess, (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun), req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC); diff --git a/iscsi-scst/kernel/isert-scst/iser.h b/iscsi-scst/kernel/isert-scst/iser.h index eeb8040fc..ccf1bd66e 100644 --- a/iscsi-scst/kernel/isert-scst/iser.h +++ b/iscsi-scst/kernel/isert-scst/iser.h @@ -63,6 +63,7 @@ struct isert_portal { /* protected by dev_list_mutex */ struct list_head conn_list; /* head of conns list */ enum isert_portal_state state; + int refcnt; }; struct isert_buf { @@ -158,8 +159,11 @@ struct isert_cq { #define ISERT_CONNECTION_ABORTED 0 #define ISERT_DRAIN_POSTED 1 -#define ISERT_DRAIN_FAILED 2 -#define ISERT_DISCON_CALLED 3 +#define ISERT_DISCON_CALLED 2 +#define ISERT_DRAINED_RQ 3 +#define ISERT_DRAINED_SQ 4 +#define ISERT_CONNECTION_CLOSE 5 +#define ISERT_IN_PORTAL_LIST 6 struct isert_connection { struct iscsi_conn iscsi ____cacheline_aligned; @@ -185,6 +189,7 @@ struct isert_connection { struct isert_cq *cq_desc; enum isert_conn_state state; + struct mutex state_mutex; u32 responder_resources; u32 initiator_depth; @@ -216,7 +221,9 @@ struct isert_connection { struct work_struct close_work; struct work_struct drain_work; struct work_struct discon_work; - struct isert_wr drain_wr; + struct work_struct free_work; + struct isert_wr drain_wr_sq; + struct isert_wr drain_wr_rq; struct kref kref; struct isert_portal *portal; @@ -287,6 +294,7 @@ void isert_free_conn_resources(struct isert_connection *isert_conn); void isert_conn_free(struct isert_connection *isert_conn); void isert_conn_disconnect(struct isert_connection *isert_conn); void isert_post_drain(struct isert_connection *isert_conn); +void isert_sched_conn_free(struct isert_connection *isert_conn); static inline struct isert_connection *isert_conn_zalloc(void) { diff --git a/iscsi-scst/kernel/isert-scst/iser_buf.c b/iscsi-scst/kernel/isert-scst/iser_buf.c index 7bd34c9a2..f6bc47fab 100644 --- a/iscsi-scst/kernel/isert-scst/iser_buf.c +++ b/iscsi-scst/kernel/isert-scst/iser_buf.c @@ -52,7 +52,7 @@ static int isert_buf_alloc_pg(struct ib_device *ib_dev, isert_buf->sg = kmalloc_array(isert_buf->sg_cnt, sizeof(*isert_buf->sg), GFP_KERNEL); if (unlikely(!isert_buf->sg)) { - pr_err("Failed to allocate buffer SG\n"); + PRINT_ERROR("Failed to allocate buffer SG"); res = -ENOMEM; goto out; } @@ -63,7 +63,7 @@ static int isert_buf_alloc_pg(struct ib_device *ib_dev, page = alloc_page(GFP_KERNEL); if (unlikely(!page)) { - pr_err("Failed to allocate page\n"); + PRINT_ERROR("Failed to allocate page"); res = -ENOMEM; goto out_map_failed; } @@ -74,8 +74,8 @@ static int isert_buf_alloc_pg(struct ib_device *ib_dev, res = ib_dma_map_sg(ib_dev, isert_buf->sg, isert_buf->sg_cnt, dma_dir); if (unlikely(!res)) { --i; /* do not overrun isert_buf->sg */ - pr_err("Failed to DMA map iser sg:%p len:%d\n", - isert_buf->sg, isert_buf->sg_cnt); + PRINT_ERROR("Failed to DMA map iser sg:%p len:%d", + isert_buf->sg, isert_buf->sg_cnt); res = -ENOMEM; goto out_map_failed; } @@ -111,14 +111,14 @@ static int isert_buf_malloc(struct ib_device *ib_dev, isert_buf->sg_cnt = 1; isert_buf->sg = kmalloc(sizeof(isert_buf->sg[0]), GFP_KERNEL); if (unlikely(!isert_buf->sg)) { - pr_err("Failed to allocate buffer SG\n"); + PRINT_ERROR("Failed to allocate buffer SG"); res = -ENOMEM; goto out; } isert_buf->addr = kmalloc(size, GFP_KERNEL); if (unlikely(!isert_buf->addr)) { - pr_err("Failed to allocate data buffer\n"); + PRINT_ERROR("Failed to allocate data buffer"); res = -ENOMEM; goto data_malloc_failed; } @@ -127,8 +127,8 @@ static int isert_buf_malloc(struct ib_device *ib_dev, res = ib_dma_map_sg(ib_dev, isert_buf->sg, isert_buf->sg_cnt, dma_dir); if (unlikely(!res)) { - pr_err("Failed to DMA map iser sg:%p len:%d\n", - isert_buf->sg, isert_buf->sg_cnt); + PRINT_ERROR("Failed to DMA map iser sg:%p len:%d", + isert_buf->sg, isert_buf->sg_cnt); res = -ENOMEM; goto out_map_failed; } @@ -233,7 +233,7 @@ int isert_wr_init(struct isert_wr *wr, case ISER_WR_RDMA_READ: send_wr_op = IB_WR_RDMA_READ; if (unlikely(!pdu->is_wstag_valid)) { - pr_err("No write tag/va specified for RDMA op\n"); + PRINT_ERROR("No write tag/va specified for RDMA op"); isert_buf_release(isert_buf); buff_offset = -EFAULT; goto out; @@ -250,7 +250,7 @@ int isert_wr_init(struct isert_wr *wr, case ISER_WR_RDMA_WRITE: send_wr_op = IB_WR_RDMA_WRITE; if (unlikely(!pdu->is_rstag_valid)) { - pr_err("No read tag/va specified for RDMA op\n"); + PRINT_ERROR("No read tag/va specified for RDMA op"); isert_buf_release(isert_buf); buff_offset = -EFAULT; goto out; diff --git a/iscsi-scst/kernel/isert-scst/iser_datamover.c b/iscsi-scst/kernel/isert-scst/iser_datamover.c index 165ab071b..7dd723dbc 100644 --- a/iscsi-scst/kernel/isert-scst/iser_datamover.c +++ b/iscsi-scst/kernel/isert-scst/iser_datamover.c @@ -47,7 +47,7 @@ int isert_datamover_init(void) err = isert_global_init(); if (unlikely(err)) { - pr_err("iser datamover init failed, err:%d\n", err); + PRINT_ERROR("iser datamover init failed, err:%d", err); return err; } return 0; @@ -189,7 +189,7 @@ int isert_login_rsp_tx(struct iscsi_cmnd *login_rsp, int last, int discovery) if (last && !discovery) { err = isert_alloc_conn_resources(isert_conn); if (unlikely(err)) { - pr_err("Failed to init conn resources\n"); + PRINT_ERROR("Failed to init conn resources"); return err; } isert_pdu_free(isert_conn->login_req_pdu); @@ -199,7 +199,7 @@ int isert_login_rsp_tx(struct iscsi_cmnd *login_rsp, int last, int discovery) &isert_conn->login_req_pdu->wr[0], 1); if (unlikely(err)) { - pr_err("Failed to post recv login req rx buf, err:%d\n", err); + PRINT_ERROR("Failed to post recv login req rx buf, err:%d", err); return err; } } diff --git a/iscsi-scst/kernel/isert-scst/iser_datamover.h b/iscsi-scst/kernel/isert-scst/iser_datamover.h index 9a41dc291..6191fcd10 100644 --- a/iscsi-scst/kernel/isert-scst/iser_datamover.h +++ b/iscsi-scst/kernel/isert-scst/iser_datamover.h @@ -86,7 +86,8 @@ int isert_data_in_sent(struct iscsi_cmnd *cmd); int isert_pdu_sent(struct iscsi_cmnd *pdu); void isert_pdu_err(struct iscsi_cmnd *pdu); -int isert_connection_closed(struct iscsi_conn *iscsi_conn); +void isert_connection_closed(struct iscsi_conn *iscsi_conn); +void isert_connection_abort(struct iscsi_conn *iscsi_conn); void *isert_get_priv(struct iscsi_conn *iscsi_conn); void isert_set_priv(struct iscsi_conn *iscsi_conn, void *priv); diff --git a/iscsi-scst/kernel/isert-scst/iser_global.c b/iscsi-scst/kernel/isert-scst/iser_global.c index 31fe1f306..2c0c92410 100644 --- a/iscsi-scst/kernel/isert-scst/iser_global.c +++ b/iscsi-scst/kernel/isert-scst/iser_global.c @@ -106,7 +106,7 @@ int isert_global_init(void) isert_glob.conn_wq = create_workqueue("isert_conn_wq"); if (!isert_glob.conn_wq) { - pr_err("Failed to alloc iser conn work queue\n"); + PRINT_ERROR("Failed to alloc iser conn work queue"); return -ENOMEM; } @@ -114,7 +114,7 @@ int isert_global_init(void) SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); if (!isert_cmnd_cache) { destroy_workqueue(isert_glob.conn_wq); - pr_err("Failed to alloc iser command cache\n"); + PRINT_ERROR("Failed to alloc iser command cache"); return -ENOMEM; } @@ -123,7 +123,7 @@ int isert_global_init(void) if (!isert_conn_cache) { destroy_workqueue(isert_glob.conn_wq); kmem_cache_destroy(isert_cmnd_cache); - pr_err("Failed to alloc iser connection cache\n"); + PRINT_ERROR("Failed to alloc iser connection cache"); return -ENOMEM; } @@ -153,7 +153,7 @@ int isert_get_addr_size(struct sockaddr *sa, size_t *addr_len) *addr_len = sizeof(struct sockaddr_in6); break; default: - pr_err("Unknown address family\n"); + PRINT_ERROR("Unknown address family"); ret = -EINVAL; goto out; } diff --git a/iscsi-scst/kernel/isert-scst/iser_pdu.c b/iscsi-scst/kernel/isert-scst/iser_pdu.c index cf30db258..177b71f3b 100644 --- a/iscsi-scst/kernel/isert-scst/iser_pdu.c +++ b/iscsi-scst/kernel/isert-scst/iser_pdu.c @@ -263,8 +263,8 @@ int isert_prepare_rdma(struct isert_cmnd *isert_pdu, err = ib_dma_map_sg(ib_dev, isert_buf->sg, isert_buf->sg_cnt, isert_buf->dma_dir); if (unlikely(!err)) { - pr_err("Failed to DMA map iser sg:%p len:%d\n", - isert_buf->sg, isert_buf->sg_cnt); + PRINT_ERROR("Failed to DMA map iser sg:%p len:%d", + isert_buf->sg, isert_buf->sg_cnt); wr_cnt = -EFAULT; goto out; } @@ -331,27 +331,27 @@ struct isert_cmnd *isert_rx_pdu_alloc(struct isert_connection *isert_conn, pdu = isert_pdu_alloc(); if (unlikely(!pdu)) { - pr_err("Failed to alloc pdu\n"); + PRINT_ERROR("Failed to alloc pdu"); goto out; } err = isert_alloc_for_rdma(pdu, 4, isert_conn); if (unlikely(err)) { - pr_err("Failed to alloc sge and wr for rx pdu\n"); + PRINT_ERROR("Failed to alloc sge and wr for rx pdu"); goto out; } err = isert_buf_alloc_data_buf(isert_conn->isert_dev->ib_dev, &pdu->buf, size, DMA_FROM_DEVICE); if (unlikely(err)) { - pr_err("Failed to alloc rx pdu buf sz:%zd\n", size); + PRINT_ERROR("Failed to alloc rx pdu buf sz:%zd", size); goto buf_alloc_failed; } err = isert_rx_pdu_init(pdu, isert_conn); if (unlikely(err)) { - pr_err("Failed to init rx pdu wr:%p size:%zd err:%d\n", - &pdu->wr, size, err); + PRINT_ERROR("Failed to init rx pdu wr:%p size:%zd err:%d", + &pdu->wr, size, err); goto pdu_init_failed; } @@ -379,27 +379,27 @@ struct isert_cmnd *isert_tx_pdu_alloc(struct isert_connection *isert_conn, pdu = isert_pdu_alloc(); if (unlikely(!pdu)) { - pr_err("Failed to alloc pdu\n"); + PRINT_ERROR("Failed to alloc pdu"); goto out; } err = isert_alloc_for_rdma(pdu, 4, isert_conn); if (unlikely(err)) { - pr_err("Failed to alloc sge and wr for tx pdu\n"); + PRINT_ERROR("Failed to alloc sge and wr for tx pdu"); goto out; } err = isert_buf_alloc_data_buf(isert_conn->isert_dev->ib_dev, &pdu->buf, size, DMA_TO_DEVICE); if (unlikely(err)) { - pr_err("Failed to alloc tx pdu buf sz:%zd\n", size); + PRINT_ERROR("Failed to alloc tx pdu buf sz:%zd", size); goto buf_alloc_failed; } err = isert_pdu_tx_buf_init(pdu, isert_conn); if (unlikely(err < 0)) { - pr_err("Failed to init tx pdu wr:%p size:%zd err:%d\n", - &pdu->wr, size, err); + PRINT_ERROR("Failed to init tx pdu wr:%p size:%zd err:%d", + &pdu->wr, size, err); goto buf_init_failed; } @@ -449,8 +449,8 @@ int isert_alloc_conn_resources(struct isert_connection *isert_conn) to_alloc = isert_conn->queue_depth * 2 + isert_conn->repost_threshold; if (unlikely(to_alloc > ISER_MAX_WCE)) { - pr_err("QueuedCommands larger than %d not supported\n", - (ISER_MAX_WCE - isert_conn->repost_threshold) / 2); + PRINT_ERROR("QueuedCommands larger than %d not supported", + (ISER_MAX_WCE - isert_conn->repost_threshold) / 2); err = -EINVAL; goto out; } @@ -478,7 +478,7 @@ int isert_alloc_conn_resources(struct isert_connection *isert_conn) err = isert_post_recv(isert_conn, &first_pdu->wr[0], to_alloc); if (unlikely(err)) { - pr_err("Failed to post recv err:%d\n", err); + PRINT_ERROR("Failed to post recv err:%d", err); goto clean_pdus; } @@ -596,8 +596,8 @@ int isert_pdu_send(struct isert_connection *isert_conn, err = isert_post_send(isert_conn, wr, 1); if (unlikely(err)) { - pr_err("Failed to send pdu conn:%p pdu:%p err:%d\n", - isert_conn, tx_pdu, err); + PRINT_ERROR("Failed to send pdu conn:%p pdu:%p err:%d", + isert_conn, tx_pdu, err); } TRACE_EXIT_RES(err); @@ -623,8 +623,8 @@ int isert_pdu_post_rdma_write(struct isert_connection *isert_conn, isert_link_send_pdu_wrs(isert_cmd, isert_rsp, wr_cnt); err = isert_post_send(isert_conn, &isert_cmd->wr[0], wr_cnt + 1); if (unlikely(err)) { - pr_err("Failed to send pdu conn:%p pdu:%p err:%d\n", - isert_conn, isert_cmd, err); + PRINT_ERROR("Failed to send pdu conn:%p pdu:%p err:%d", + isert_conn, isert_cmd, err); } TRACE_EXIT_RES(err); @@ -640,8 +640,8 @@ int isert_pdu_post_rdma_read(struct isert_connection *isert_conn, err = isert_post_send(isert_conn, &isert_cmd->wr[0], wr_cnt); if (unlikely(err)) { - pr_err("Failed to send pdu conn:%p pdu:%p err:%d\n", - isert_conn, isert_cmd, err); + PRINT_ERROR("Failed to send pdu conn:%p pdu:%p err:%d", + isert_conn, isert_cmd, err); } TRACE_EXIT_RES(err); diff --git a/iscsi-scst/kernel/isert-scst/iser_rdma.c b/iscsi-scst/kernel/isert-scst/iser_rdma.c index 9f50728d9..9a48679a2 100644 --- a/iscsi-scst/kernel/isert-scst/iser_rdma.c +++ b/iscsi-scst/kernel/isert-scst/iser_rdma.c @@ -75,13 +75,20 @@ int isert_post_recv(struct isert_connection *isert_conn, TRACE_ENTRY(); +#ifdef CONFIG_SCST_EXTRACHECKS + if (test_bit(ISERT_DRAIN_POSTED, &isert_conn->flags)) { + PRINT_ERROR("conn:%p post recv after drain", isert_conn); + BUG(); + } +#endif + err = ib_post_recv(isert_conn->qp, first_ib_wr, &bad_wr); if (unlikely(err)) { num_posted = isert_num_recv_posted_on_err(first_ib_wr, bad_wr); - pr_err("conn:%p recv posted:%d/%d 1st wr_id:0x%llx sz:%d err:%d\n", - isert_conn, num_posted, num_wr, first_ib_wr->wr_id, - first_ib_wr->sg_list->length, err); + PRINT_ERROR("conn:%p recv posted:%d/%d 1st wr_id:0x%llx sz:%d err:%d", + isert_conn, num_posted, num_wr, first_ib_wr->wr_id, + first_ib_wr->sg_list->length, err); } TRACE_EXIT_RES(err); @@ -115,63 +122,103 @@ int isert_post_send(struct isert_connection *isert_conn, TRACE_ENTRY(); +#ifdef CONFIG_SCST_EXTRACHECKS + if (test_bit(ISERT_DRAIN_POSTED, &isert_conn->flags)) { + PRINT_ERROR("conn:%p post send after drain", isert_conn); + BUG(); + } +#endif + err = ib_post_send(isert_conn->qp, first_ib_wr, &bad_wr); if (unlikely(err)) { num_posted = isert_num_send_posted_on_err(first_ib_wr, bad_wr); - pr_err("conn:%p send posted:%d/%d bad wr_id:0x%llx sz:%d num_sge: %d err:%d\n", - isert_conn, num_posted, num_wr, bad_wr->wr_id, - bad_wr->sg_list->length, bad_wr->num_sge, err); + PRINT_ERROR("conn:%p send posted:%d/%d bad wr_id:0x%llx sz:%d num_sge: %d err:%d", + isert_conn, num_posted, num_wr, bad_wr->wr_id, + bad_wr->sg_list->length, bad_wr->num_sge, err); } TRACE_EXIT_RES(err); return err; } +static void isert_post_drain_sq(struct isert_connection* isert_conn) +{ + struct ib_send_wr* bad_wr; + struct isert_wr *drain_wr_sq = &isert_conn->drain_wr_sq; + int err; + + isert_wr_set_fields(drain_wr_sq, isert_conn, NULL); + drain_wr_sq->wr_op = ISER_WR_SEND; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + drain_wr_sq->send_wr.wr_id = _ptr_to_u64(drain_wr_sq); + drain_wr_sq->send_wr.opcode = IB_WR_SEND; + err = ib_post_send(isert_conn->qp, + &drain_wr_sq->send_wr, &bad_wr); +#else + drain_wr_sq->send_wr.wr.wr_id = _ptr_to_u64(drain_wr_sq); + drain_wr_sq->send_wr.wr.opcode = IB_WR_SEND; + err = ib_post_send(isert_conn->qp, + &drain_wr_sq->send_wr.wr, &bad_wr); +#endif + if (unlikely(err)) { + PRINT_ERROR("Failed to post drain wr to send queue, err:%d", err); + /* We need to decrement iser_conn->kref in order to be able to cleanup + * the connection */ + set_bit(ISERT_DRAINED_SQ, &isert_conn->flags); + if (test_bit(ISERT_DRAINED_RQ, &isert_conn->flags)) { + isert_sched_conn_free(isert_conn); + } + } +} + +static void isert_post_drain_rq(struct isert_connection *isert_conn) +{ + struct ib_recv_wr *bad_wr; + struct isert_wr *drain_wr_rq = &isert_conn->drain_wr_rq; + int err; + + isert_wr_set_fields(drain_wr_rq, isert_conn, NULL); + drain_wr_rq->wr_op = ISER_WR_RECV; + drain_wr_rq->recv_wr.wr_id = _ptr_to_u64(drain_wr_rq); + err = ib_post_recv(isert_conn->qp, + &drain_wr_rq->recv_wr, &bad_wr); + if (unlikely(err)) { + PRINT_ERROR("Failed to post drain wr to receive queue, err:%d", err); + set_bit(ISERT_DRAINED_RQ, &isert_conn->flags); + if (test_bit(ISERT_DRAINED_SQ, &isert_conn->flags)) { + isert_sched_conn_free(isert_conn); + } + } +} + void isert_post_drain(struct isert_connection *isert_conn) { if (!test_and_set_bit(ISERT_DRAIN_POSTED, &isert_conn->flags)) { - struct ib_send_wr *bad_wr; - int err; - - isert_wr_set_fields(&isert_conn->drain_wr, isert_conn, NULL); - isert_conn->drain_wr.wr_op = ISER_WR_SEND; -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) - isert_conn->drain_wr.send_wr.wr_id = - _ptr_to_u64(&isert_conn->drain_wr); - isert_conn->drain_wr.send_wr.opcode = IB_WR_SEND; - err = ib_post_send(isert_conn->qp, - &isert_conn->drain_wr.send_wr, &bad_wr); -#else - isert_conn->drain_wr.send_wr.wr.wr_id = - _ptr_to_u64(&isert_conn->drain_wr); - isert_conn->drain_wr.send_wr.wr.opcode = IB_WR_SEND; - err = ib_post_send(isert_conn->qp, - &isert_conn->drain_wr.send_wr.wr, &bad_wr); -#endif - if (unlikely(err)) { - pr_err("Failed to post drain wr, err:%d\n", err); - /* - * We need to decrement iser_conn->kref in order to be - * able to cleanup the connection. - */ - set_bit(ISERT_DRAIN_FAILED, &isert_conn->flags); - isert_conn_free(isert_conn); - } + isert_post_drain_rq(isert_conn); + isert_post_drain_sq(isert_conn); } } void isert_conn_disconnect(struct isert_connection *isert_conn) { - int err = rdma_disconnect(isert_conn->cm_id); + int err; - if (unlikely(err)) - pr_err("Failed to rdma disconnect, err:%d\n", err); + mutex_lock(&isert_conn->state_mutex); + if (isert_conn->state != ISER_CONN_CLOSING) { + isert_conn->state = ISER_CONN_CLOSING; + + err = rdma_disconnect(isert_conn->cm_id); + + if (unlikely(err)) + PRINT_ERROR("Failed to rdma disconnect, err:%d", err); + } + mutex_unlock(&isert_conn->state_mutex); } static int isert_pdu_handle_hello_req(struct isert_cmnd *pdu) { - pr_info("iSER Hello not supported\n"); + PRINT_INFO("iSER Hello not supported"); return -EINVAL; /* meanwhile disconnect immediately */ } @@ -206,7 +253,7 @@ static int isert_pdu_handle_tm_func(struct isert_cmnd *pdu) static int isert_pdu_handle_data_out(struct isert_cmnd *pdu) { - pr_info("iser iscsi data out not supported\n"); + PRINT_INFO("iser iscsi data out not supported"); return -EINVAL; /* meanwhile disconnect immediately */ } @@ -217,7 +264,7 @@ static int isert_pdu_handle_logout(struct isert_cmnd *pdu) static int isert_pdu_handle_snack(struct isert_cmnd *pdu) { - pr_info("iser iscsi SNACK not supported\n"); + PRINT_INFO("iser iscsi SNACK not supported"); return -EINVAL; /* meanwhile disconnect immediately */ } @@ -340,8 +387,8 @@ static void isert_recv_completion_handler(struct isert_wr *wr) err = isert_pdu_handle_snack(pdu); break; default: - pr_err("Unexpected iscsi opcode:0x%x\n", - pdu->iscsi_opcode); + PRINT_ERROR("Unexpected iscsi opcode:0x%x", + pdu->iscsi_opcode); err = -EINVAL; break; } @@ -350,14 +397,14 @@ static void isert_recv_completion_handler(struct isert_wr *wr) err = isert_pdu_handle_hello_req(pdu); break; default: - pr_err("malformed isert_hdr, iser op:%x flags 0x%02x\n", - pdu->isert_opcode, pdu->isert_hdr->flags); + PRINT_ERROR("malformed isert_hdr, iser op:%x flags 0x%02x", + pdu->isert_opcode, pdu->isert_hdr->flags); err = -EINVAL; break; } if (unlikely(err)) { - pr_err("err:%d while handling iser pdu\n", err); + PRINT_ERROR("err:%d while handling iser pdu", err); isert_conn_disconnect(wr->conn); } @@ -423,7 +470,7 @@ static void isert_handle_wc(struct ib_wc *wc) if (unlikely(isert_conn->state == ISER_CONN_HANDSHAKE)) { isert_conn->state = ISER_CONN_ACTIVE; isert_conn->saved_wr = wr; - pr_info("iser rx pdu before conn established, pdu saved\n"); + PRINT_INFO("iser rx pdu before conn established, pdu saved"); break; } isert_recv_completion_handler(wr); @@ -439,8 +486,8 @@ static void isert_handle_wc(struct ib_wc *wc) break; default: isert_conn = wr->conn; - pr_err("unexpected work req op:%d, wc op:%d, wc:%p wr_id:%p conn:%p\n", - wr->wr_op, wc->opcode, wc, wr, isert_conn); + PRINT_ERROR("unexpected work req op:%d, wc op:%d, wc:%p wr_id:%p conn:%p", + wr->wr_op, wc->opcode, wc, wr, isert_conn); if (isert_conn) isert_conn_disconnect(isert_conn); break; @@ -590,8 +637,7 @@ static void isert_conn_closed_do_work(struct work_struct *work) #endif if (!test_bit(ISERT_CONNECTION_ABORTED, &isert_conn->flags)) - if (!test_and_set_bit(ISERT_DISCON_CALLED, &isert_conn->flags)) - isert_connection_closed(&isert_conn->iscsi); + isert_connection_abort(&isert_conn->iscsi); isert_conn_free(isert_conn); } @@ -607,6 +653,32 @@ static void isert_sched_conn_closed(struct isert_connection *isert_conn) isert_conn_queue_work(&isert_conn->close_work); } +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) +static void isert_conn_free_do_work(void *ctx) +#else +static void isert_conn_free_do_work(struct work_struct *work) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) + struct isert_connection *isert_conn = ctx; +#else + struct isert_connection *isert_conn = + container_of(work, struct isert_connection, free_work); +#endif + + isert_conn_free(isert_conn); +} + +void isert_sched_conn_free(struct isert_connection *isert_conn) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) + INIT_WORK(&isert_conn->free_work, isert_conn_free_do_work, + isert_conn); +#else + INIT_WORK(&isert_conn->free_work, isert_conn_free_do_work); +#endif + isert_conn_queue_work(&isert_conn->free_work); +} static void isert_handle_wc_error(struct ib_wc *wc) { @@ -621,9 +693,9 @@ static void isert_handle_wc_error(struct ib_wc *wc) TRACE_ENTRY(); if (wc->status != IB_WC_WR_FLUSH_ERR) - pr_err("conn:%p wr_id:0x%p status:%s vendor_err:0x%0x\n", - isert_conn, wr, wr_status_str(wc->status), - wc->vendor_err); + PRINT_ERROR("conn:%p wr_id:0x%p status:%s vendor_err:0x%0x", + isert_conn, wr, wr_status_str(wc->status), + wc->vendor_err); if (!test_bit(ISERT_CONNECTION_ABORTED, &isert_conn->flags)) if (!test_and_set_bit(ISERT_DISCON_CALLED, &isert_conn->flags)) @@ -636,8 +708,12 @@ static void isert_handle_wc_error(struct ib_wc *wc) #else num_sge = wr->send_wr.wr.num_sge; #endif - if (unlikely(num_sge == 0)) /* Drain WR */ - isert_sched_conn_drained(isert_conn); + if (unlikely(num_sge == 0)) { /* Drain WR */ + set_bit(ISERT_DRAINED_SQ, &isert_conn->flags); + if (test_bit(ISERT_DRAINED_RQ, &isert_conn->flags)) { + isert_sched_conn_drained(isert_conn); + } + } else if (!isert_pdu->is_fake_rx) isert_pdu_err(&isert_pdu->iscsi); break; @@ -652,6 +728,13 @@ static void isert_handle_wc_error(struct ib_wc *wc) break; case ISER_WR_RECV: /* this should be the Flush, no task has been created yet */ + num_sge = wr->recv_wr.num_sge; + if (unlikely(num_sge == 0)) { /* Drain WR */ + set_bit(ISERT_DRAINED_RQ, &isert_conn->flags); + if (test_bit(ISERT_DRAINED_SQ, &isert_conn->flags)) { + isert_sched_conn_drained(isert_conn); + } + } break; case ISER_WR_RDMA_WRITE: if (isert_buf->sg_cnt != 0) { @@ -666,8 +749,8 @@ static void isert_handle_wc_error(struct ib_wc *wc) */ break; default: - pr_err("unexpected opcode %d, wc:%p wr_id:%p conn:%p\n", - wr->wr_op, wc, wr, isert_conn); + PRINT_ERROR("unexpected opcode %d, wc:%p wr_id:%p conn:%p", + wr->wr_op, wc, wr, isert_conn); break; } @@ -715,7 +798,7 @@ static void isert_cq_comp_work_cb(struct work_struct *work) ret = isert_poll_cq(cq_desc); if (unlikely(ret < 0)) { /* poll error */ - pr_err("ib_poll_cq failed\n"); + PRINT_ERROR("ib_poll_cq failed"); goto out; } @@ -803,9 +886,9 @@ static void isert_async_evt_handler(struct ib_event *async_ev, void *context) switch (ev_type) { case IB_EVENT_COMM_EST: isert_conn = async_ev->element.qp->qp_context; - pr_info("conn:0x%p cm_id:0x%p dev:%s, QP evt: %s\n", - isert_conn, isert_conn->cm_id, dev_name, - ib_event_type_str(IB_EVENT_COMM_EST)); + PRINT_INFO("conn:0x%p cm_id:0x%p dev:%s, QP evt: %s", + isert_conn, isert_conn->cm_id, dev_name, + ib_event_type_str(IB_EVENT_COMM_EST)); /* force "connection established" event */ rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); break; @@ -819,22 +902,22 @@ static void isert_async_evt_handler(struct ib_event *async_ev, void *context) case IB_EVENT_PATH_MIG_ERR: case IB_EVENT_QP_LAST_WQE_REACHED: isert_conn = async_ev->element.qp->qp_context; - pr_err("conn:0x%p cm_id:0x%p dev:%s, QP evt: %s\n", - isert_conn, isert_conn->cm_id, dev_name, - ib_event_type_str(ev_type)); + PRINT_ERROR("conn:0x%p cm_id:0x%p dev:%s, QP evt: %s", + isert_conn, isert_conn->cm_id, dev_name, + ib_event_type_str(ev_type)); break; /* CQ-related events */ case IB_EVENT_CQ_ERR: - pr_err("dev:%s CQ evt: %s\n", dev_name, - ib_event_type_str(ev_type)); + PRINT_ERROR("dev:%s CQ evt: %s", dev_name, + ib_event_type_str(ev_type)); break; /* SRQ events */ case IB_EVENT_SRQ_ERR: case IB_EVENT_SRQ_LIMIT_REACHED: - pr_err("dev:%s SRQ evt: %s\n", dev_name, - ib_event_type_str(ev_type)); + PRINT_ERROR("dev:%s SRQ evt: %s", dev_name, + ib_event_type_str(ev_type)); break; /* Port events */ @@ -844,20 +927,20 @@ static void isert_async_evt_handler(struct ib_event *async_ev, void *context) case IB_EVENT_PKEY_CHANGE: case IB_EVENT_SM_CHANGE: case IB_EVENT_CLIENT_REREGISTER: - pr_err("dev:%s port:%d evt: %s\n", - dev_name, async_ev->element.port_num, - ib_event_type_str(ev_type)); + PRINT_ERROR("dev:%s port:%d evt: %s", + dev_name, async_ev->element.port_num, + ib_event_type_str(ev_type)); break; /* HCA events */ case IB_EVENT_DEVICE_FATAL: - pr_err("dev:%s HCA evt: %s\n", dev_name, - ib_event_type_str(ev_type)); + PRINT_ERROR("dev:%s HCA evt: %s", dev_name, + ib_event_type_str(ev_type)); break; default: - pr_err("dev:%s evt: %s\n", dev_name, - ib_event_type_str(ev_type)); + PRINT_ERROR("dev:%s evt: %s", dev_name, + ib_event_type_str(ev_type)); break; } @@ -878,7 +961,7 @@ static struct isert_device *isert_device_create(struct ib_device *ib_dev) isert_dev = kzalloc(sizeof(*isert_dev), GFP_KERNEL); if (unlikely(isert_dev == NULL)) { - pr_err("Failed to allocate iser dev\n"); + PRINT_ERROR("Failed to allocate iser dev"); err = -ENOMEM; goto out; } @@ -886,7 +969,7 @@ static struct isert_device *isert_device_create(struct ib_device *ib_dev) #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) err = ib_query_device(ib_dev, &isert_dev->device_attr); if (unlikely(err)) { - pr_err("Failed to query device, err: %d\n", err); + PRINT_ERROR("Failed to query device, err: %d", err); goto free_isert_dev; } #else @@ -900,15 +983,15 @@ static struct isert_device *isert_device_create(struct ib_device *ib_dev) sizeof(*isert_dev->cq_qps), GFP_KERNEL); if (unlikely(isert_dev->cq_qps == NULL)) { - pr_err("Failed to allocate iser cq_qps\n"); + PRINT_ERROR("Failed to allocate iser cq_qps"); err = -ENOMEM; goto free_isert_dev; } isert_dev->cq_desc = vmalloc(sizeof(*isert_dev->cq_desc) * isert_dev->num_cqs); if (unlikely(isert_dev->cq_desc == NULL)) { - pr_err("Failed to allocate %ld bytes for iser cq_desc\n", - sizeof(*isert_dev->cq_desc) * isert_dev->num_cqs); + PRINT_ERROR("Failed to allocate %ld bytes for iser cq_desc", + sizeof(*isert_dev->cq_desc) * isert_dev->num_cqs); err = -ENOMEM; goto fail_alloc_cq_desc; } @@ -916,14 +999,14 @@ static struct isert_device *isert_device_create(struct ib_device *ib_dev) pd = ib_alloc_pd(ib_dev); if (unlikely(IS_ERR(pd))) { err = PTR_ERR(pd); - pr_err("Failed to alloc iser dev pd, err:%d\n", err); + PRINT_ERROR("Failed to alloc iser dev pd, err:%d", err); goto fail_pd; } mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE); if (unlikely(IS_ERR(mr))) { err = PTR_ERR(mr); - pr_err("Failed to get dma mr, err: %d\n", err); + PRINT_ERROR("Failed to get dma mr, err: %d", err); goto fail_mr; } @@ -932,7 +1015,7 @@ static struct isert_device *isert_device_create(struct ib_device *ib_dev) #ifdef CONFIG_SCST_EXTRACHECKS if (isert_dev->device_attr.max_cqe == 0) - pr_err("Zero max_cqe encountered: you may have a compilation problem\n"); + PRINT_ERROR("Zero max_cqe encountered: you may have a compilation problem"); #endif for (i = 0; i < isert_dev->num_cqs; ++i) { @@ -961,8 +1044,8 @@ static struct isert_device *isert_device_create(struct ib_device *ib_dev) #endif #endif if (unlikely(!cq_desc->cq_workqueue)) { - pr_err("Failed to alloc iser cq work queue for dev:%s\n", - ib_dev->name); + PRINT_ERROR("Failed to alloc iser cq work queue for dev:%s", + ib_dev->name); err = -ENOMEM; goto fail_cq; } @@ -991,14 +1074,14 @@ static struct isert_device *isert_device_create(struct ib_device *ib_dev) if (unlikely(IS_ERR(cq))) { cq_desc->cq = NULL; err = PTR_ERR(cq); - pr_err("Failed to create iser dev cq, err:%d\n", err); + PRINT_ERROR("Failed to create iser dev cq, err:%d", err); goto fail_cq; } cq_desc->cq = cq; err = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); if (unlikely(err)) { - pr_err("Failed to request notify cq, err: %d\n", err); + PRINT_ERROR("Failed to request notify cq, err: %d", err); goto fail_cq; } } @@ -1013,7 +1096,7 @@ static struct isert_device *isert_device_create(struct ib_device *ib_dev) isert_dev_list_add(isert_dev); - pr_info("iser created device:%p\n", isert_dev); + PRINT_INFO("iser created device:%p", isert_dev); return isert_dev; fail_cq: @@ -1062,14 +1145,14 @@ static void isert_device_release(struct isert_device *isert_dev) err = ib_destroy_cq(cq_desc->cq); if (unlikely(err)) - pr_err("Failed to destroy cq, err:%d\n", err); + PRINT_ERROR("Failed to destroy cq, err:%d", err); destroy_workqueue(cq_desc->cq_workqueue); } err = ib_dereg_mr(isert_dev->mr); if (unlikely(err)) - pr_err("Failed to destroy mr, err:%d\n", err); + PRINT_ERROR("Failed to destroy mr, err:%d", err); ib_dealloc_pd(isert_dev->pd); vfree(isert_dev->cq_desc); @@ -1127,7 +1210,7 @@ static int isert_conn_qp_create(struct isert_connection *isert_conn) do { if (max_wr < ISER_MIN_SQ_SIZE) { - pr_err("Failed to create qp, not enough memory\n"); + PRINT_ERROR("Failed to create qp, not enough memory"); goto fail_create_qp; } @@ -1136,7 +1219,7 @@ static int isert_conn_qp_create(struct isert_connection *isert_conn) err = rdma_create_qp(cm_id, isert_dev->pd, &qp_attr); if (err && err != -ENOMEM) { - pr_err("Failed to create qp, err:%d\n", err); + PRINT_ERROR("Failed to create qp, err:%d", err); goto fail_create_qp; } @@ -1145,7 +1228,7 @@ static int isert_conn_qp_create(struct isert_connection *isert_conn) isert_conn->qp = cm_id->qp; - pr_info("iser created cm_id:%p qp:0x%X\n", cm_id, cm_id->qp->qp_num); + PRINT_INFO("iser created cm_id:%p qp:0x%X", cm_id, cm_id->qp->qp_num); out: TRACE_EXIT_RES(err); @@ -1169,7 +1252,7 @@ static struct isert_connection *isert_conn_create(struct rdma_cm_id *cm_id, isert_conn = isert_conn_zalloc(); if (unlikely(!isert_conn)) { - pr_err("Unable to allocate iser conn, cm_id:%p\n", cm_id); + PRINT_ERROR("Unable to allocate iser conn, cm_id:%p", cm_id); err = -ENOMEM; goto fail_alloc; } @@ -1201,7 +1284,7 @@ static struct isert_connection *isert_conn_create(struct rdma_cm_id *cm_id, isert_conn->login_req_pdu = isert_rx_pdu_alloc(isert_conn, ISER_MAX_LOGIN_RDSL); if (unlikely(!isert_conn->login_req_pdu)) { - pr_err("Failed to init login req rx pdu\n"); + PRINT_ERROR("Failed to init login req rx pdu"); err = -ENOMEM; goto fail_login_req_pdu; } @@ -1209,7 +1292,7 @@ static struct isert_connection *isert_conn_create(struct rdma_cm_id *cm_id, isert_conn->login_rsp_pdu = isert_tx_pdu_alloc(isert_conn, ISER_MAX_LOGIN_RDSL); if (unlikely(!isert_conn->login_rsp_pdu)) { - pr_err("Failed to init login rsp tx pdu\n"); + PRINT_ERROR("Failed to init login rsp tx pdu"); err = -ENOMEM; goto fail_login_rsp_pdu; } @@ -1220,11 +1303,12 @@ static struct isert_connection *isert_conn_create(struct rdma_cm_id *cm_id, err = isert_post_recv(isert_conn, &isert_conn->login_req_pdu->wr[0], 1); if (unlikely(err)) { - pr_err("Failed to post recv login req rx buf, err:%d\n", err); + PRINT_ERROR("Failed to post recv login req rx buf, err:%d", err); goto fail_post_recv; } kref_init(&isert_conn->kref); + mutex_init(&isert_conn->state_mutex); TRACE_EXIT(); return isert_conn; @@ -1242,7 +1326,6 @@ fail_login_rsp_pdu: fail_login_req_pdu: isert_conn_kfree(isert_conn); fail_alloc: - module_put(THIS_MODULE); TRACE_EXIT_RES(err); return ERR_PTR(err); } @@ -1265,22 +1348,29 @@ static void isert_kref_free(struct kref *kref) TRACE_ENTRY(); - pr_info("isert_conn_free conn:%p\n", isert_conn); + PRINT_INFO("free conn:%p", isert_conn); isert_free_conn_resources(isert_conn); rdma_destroy_id(isert_conn->cm_id); + isert_conn->cm_id = NULL; dev = isert_get_priv(&isert_conn->iscsi); - if (dev) + if (dev) { isert_del_timer(dev); + set_bit(ISERT_CONN_PASSED, &dev->flags); + dev->conn = NULL; + dev->state = CS_DISCONNECTED; + } ib_destroy_qp(isert_conn->qp); isert_conn->qp = NULL; mutex_lock(&dev_list_mutex); + isert_conn->portal->refcnt--; isert_dev->cq_qps[cq->idx]--; - list_del(&isert_conn->portal_node); + if (test_bit(ISERT_IN_PORTAL_LIST, &isert_conn->flags)) + list_del(&isert_conn->portal_node); isert_deref_device(isert_dev); if (unlikely(isert_conn->portal->state == ISERT_PORTAL_INACTIVE)) isert_portal_free(isert_conn->portal); @@ -1295,18 +1385,34 @@ static void isert_kref_free(struct kref *kref) void isert_conn_free(struct isert_connection *isert_conn) { + sBUG_ON(atomic_read(&isert_conn->kref.refcount) == 0); kref_put(&isert_conn->kref, isert_kref_free); } -static int isert_cm_timewait_exit_handler(struct rdma_cm_id *cm_id, +static int isert_cm_disconnected_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event) { struct isert_connection *isert_conn = cm_id->qp->qp_context; - - isert_sched_conn_closed(isert_conn); + if (!test_and_set_bit(ISERT_CONNECTION_CLOSE, &isert_conn->flags)) + isert_sched_conn_closed(isert_conn); return 0; } +static void isert_immediate_conn_close(struct isert_connection* isert_conn) +{ + set_bit(ISERT_CONNECTION_ABORTED, &isert_conn->flags); + set_bit(ISERT_CONNECTION_CLOSE, &isert_conn->flags); + isert_conn->state = ISER_CONN_CLOSING; + /* + * reaching here must be with the isert_conn refcount of 3, + * one from the init and two from the connect request, + * thus it is safe to deref directly before the sched_conn_free. + */ + isert_conn_free(isert_conn); + isert_conn_free(isert_conn); + isert_sched_conn_free(isert_conn); +} + static int isert_cm_conn_req_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event) { @@ -1324,7 +1430,7 @@ static int isert_cm_conn_req_handler(struct rdma_cm_id *cm_id, if (unlikely(!try_module_get(THIS_MODULE))) { err = -EINVAL; - goto fail_get; + goto out; } mutex_lock(&dev_list_mutex); @@ -1350,7 +1456,7 @@ static int isert_cm_conn_req_handler(struct rdma_cm_id *cm_id, isert_conn->portal = portal; mutex_lock(&dev_list_mutex); - list_add_tail(&isert_conn->portal_node, &portal->conn_list); + portal->refcnt++; mutex_unlock(&dev_list_mutex); /* initiator is dst, target is src */ @@ -1374,52 +1480,57 @@ static int isert_cm_conn_req_handler(struct rdma_cm_id *cm_id, tgt_conn_param.private_data = &cm_hdr; cm_hdr.flags = ISER_ZBVA_NOT_SUPPORTED | ISER_SEND_W_INV_NOT_SUPPORTED; + kref_get(&isert_conn->kref); kref_get(&isert_conn->kref); err = rdma_accept(cm_id, &tgt_conn_param); if (unlikely(err)) { - pr_err("Failed to accept conn request, err:%d\n", err); + PRINT_ERROR("Failed to accept conn request, err:%d conn:%p", err, isert_conn); goto fail_accept; } switch (isert_conn->peer_addr.ss_family) { case AF_INET: #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 33) - pr_info("iser accepted connection cm_id:%p " - NIPQUAD_FMT "->" NIPQUAD_FMT "\n", cm_id, - NIPQUAD(((struct sockaddr_in *)&isert_conn->peer_addr)->sin_addr.s_addr), - NIPQUAD(((struct sockaddr_in *)&isert_conn->self_addr)->sin_addr.s_addr)); + PRINT_INFO("iser accepted connection cm_id:%p " + NIPQUAD_FMT "->" NIPQUAD_FMT, cm_id, + NIPQUAD(((struct sockaddr_in *)&isert_conn->peer_addr)->sin_addr.s_addr), + NIPQUAD(((struct sockaddr_in *)&isert_conn->self_addr)->sin_addr.s_addr)); #else - pr_info("iser accepted connection cm_id:%p " - "%pI4->%pI4\n", cm_id, - &((struct sockaddr_in *)&isert_conn->peer_addr)->sin_addr.s_addr, - &((struct sockaddr_in *)&isert_conn->self_addr)->sin_addr.s_addr); + PRINT_INFO("iser accepted connection cm_id:%p " + "%pI4->%pI4", cm_id, + &((struct sockaddr_in *)&isert_conn->peer_addr)->sin_addr.s_addr, + &((struct sockaddr_in *)&isert_conn->self_addr)->sin_addr.s_addr); #endif break; case AF_INET6: #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29) - pr_info("iser accepted connection cm_id:%p " - NIP6_FMT "->" NIP6_FMT "\n", cm_id, - NIP6(((struct sockaddr_in6 *)&isert_conn->peer_addr)->sin6_addr), - NIP6(((struct sockaddr_in6 *)&isert_conn->self_addr)->sin6_addr)); + PRINT_INFO("iser accepted connection cm_id:%p " + NIP6_FMT "->" NIP6_FMT, cm_id, + NIP6(((struct sockaddr_in6 *)&isert_conn->peer_addr)->sin6_addr), + NIP6(((struct sockaddr_in6 *)&isert_conn->self_addr)->sin6_addr)); #else - pr_info("iser accepted connection cm_id:%p " - "%pI6->%pI6\n", cm_id, - &((struct sockaddr_in6 *)&isert_conn->peer_addr)->sin6_addr, - &((struct sockaddr_in6 *)&isert_conn->self_addr)->sin6_addr); + PRINT_INFO("iser accepted connection cm_id:%p " + "%pI6->%pI6", cm_id, + &((struct sockaddr_in6 *)&isert_conn->peer_addr)->sin6_addr, + &((struct sockaddr_in6 *)&isert_conn->self_addr)->sin6_addr); #endif break; default: - pr_info("iser accepted connection cm_id:%p\n", cm_id); + PRINT_INFO("iser accepted connection cm_id:%p", cm_id); } + mutex_lock(&dev_list_mutex); + list_add_tail(&isert_conn->portal_node, &portal->conn_list); + set_bit(ISERT_IN_PORTAL_LIST, &isert_conn->flags); + mutex_unlock(&dev_list_mutex); + out: TRACE_EXIT_RES(err); return err; fail_accept: - set_bit(ISERT_CONNECTION_ABORTED, &isert_conn->flags); - isert_cm_timewait_exit_handler(cm_id, NULL); + isert_sched_conn_free(isert_conn); err = 0; goto out; @@ -1429,7 +1540,6 @@ fail_conn_create: mutex_unlock(&dev_list_mutex); fail_dev_create: rdma_reject(cm_id, NULL, 0); -fail_get: module_put(THIS_MODULE); goto out; } @@ -1439,42 +1549,39 @@ static int isert_cm_connect_handler(struct rdma_cm_id *cm_id, { struct isert_connection *isert_conn = cm_id->qp->qp_context; int push_saved_pdu = 0; - int ret; + int ret = 0; TRACE_ENTRY(); + mutex_lock(&isert_conn->state_mutex); if (isert_conn->state == ISER_CONN_HANDSHAKE) isert_conn->state = ISER_CONN_ACTIVE; else if (isert_conn->state == ISER_CONN_ACTIVE) push_saved_pdu = 1; + else if (isert_conn->state == ISER_CONN_CLOSING) + goto out; ret = isert_get_addr_size((struct sockaddr *)&isert_conn->peer_addr, &isert_conn->peer_addrsz); if (unlikely(ret)) goto out; - /* check if already started teardown */ - if (!unlikely(kref_get_unless_zero(&isert_conn->kref))) - goto out; - /* notify upper layer */ ret = isert_conn_established(&isert_conn->iscsi, (struct sockaddr *)&isert_conn->peer_addr, isert_conn->peer_addrsz); if (unlikely(ret)) { - set_bit(ISERT_CONNECTION_ABORTED, &isert_conn->flags); - isert_post_drain(isert_conn); - isert_conn_free(isert_conn); goto out; } if (push_saved_pdu) { - pr_info("iser push saved rx pdu\n"); + PRINT_INFO("iser push saved rx pdu"); isert_recv_completion_handler(isert_conn->saved_wr); isert_conn->saved_wr = NULL; } out: + mutex_unlock(&isert_conn->state_mutex); TRACE_EXIT_RES(ret); return ret; } @@ -1552,8 +1659,8 @@ static int isert_cm_evt_listener_handler(struct rdma_cm_id *cm_id, break; default: - pr_info("Listener event:%s(%d), ignored\n", - cm_event_type_str(ev_type), ev_type); + PRINT_INFO("Listener event:%s(%d), ignored", + cm_event_type_str(ev_type), ev_type); break; } @@ -1571,9 +1678,9 @@ static int isert_cm_evt_handler(struct rdma_cm_id *cm_id, ev_type = cm_ev->event; portal = cm_id->context; - pr_info("isert_cm_evt:%s(%d) status:%d portal:%p cm_id:%p\n", - cm_event_type_str(ev_type), ev_type, cm_ev->status, - portal, cm_id); + PRINT_INFO("isert_cm_evt:%s(%d) status:%d portal:%p cm_id:%p", + cm_event_type_str(ev_type), ev_type, cm_ev->status, + portal, cm_id); if (portal->cm_id == cm_id) { err = isert_cm_evt_listener_handler(cm_id, cm_ev); @@ -1593,21 +1700,20 @@ static int isert_cm_evt_handler(struct rdma_cm_id *cm_id, case RDMA_CM_EVENT_CONNECT_ERROR: case RDMA_CM_EVENT_REJECTED: - case RDMA_CM_EVENT_ADDR_CHANGE: - case RDMA_CM_EVENT_DISCONNECTED: err = isert_cm_disconnect_handler(cm_id, cm_ev); break; + case RDMA_CM_EVENT_ADDR_CHANGE: + case RDMA_CM_EVENT_DISCONNECTED: case RDMA_CM_EVENT_DEVICE_REMOVAL: - isert_cm_disconnect_handler(cm_id, cm_ev); - /* fallthrough */ case RDMA_CM_EVENT_TIMEWAIT_EXIT: - err = isert_cm_timewait_exit_handler(cm_id, cm_ev); + isert_cm_disconnect_handler(cm_id, cm_ev); + err = isert_cm_disconnected_handler(cm_id, cm_ev); break; case RDMA_CM_EVENT_MULTICAST_JOIN: case RDMA_CM_EVENT_MULTICAST_ERROR: - pr_err("UD-related event:%d, ignored\n", ev_type); + PRINT_ERROR("UD-related event:%d, ignored", ev_type); break; case RDMA_CM_EVENT_ADDR_RESOLVED: @@ -1615,35 +1721,31 @@ static int isert_cm_evt_handler(struct rdma_cm_id *cm_id, case RDMA_CM_EVENT_ROUTE_RESOLVED: case RDMA_CM_EVENT_ROUTE_ERROR: case RDMA_CM_EVENT_CONNECT_RESPONSE: - pr_err("Active side event:%d, ignored\n", ev_type); + PRINT_ERROR("Active side event:%d, ignored", ev_type); break; /* We can receive this instead of RDMA_CM_EVENT_ESTABLISHED */ case RDMA_CM_EVENT_UNREACHABLE: { - struct isert_connection *isert_conn; + struct isert_connection *isert_conn = cm_id->qp->qp_context; - isert_conn = cm_id->qp->qp_context; - set_bit(ISERT_CONNECTION_ABORTED, &isert_conn->flags); - /* - * reaching here must be with the isert_conn refcount of 2, - * one from the init and one from the connect request, - * thus it is safe to deref directly before the sched_conn_closed - */ - isert_conn_free(isert_conn); - isert_sched_conn_closed(isert_conn); + mutex_lock(&isert_conn->state_mutex); + if (isert_conn->state != ISER_CONN_CLOSING) { + isert_immediate_conn_close(isert_conn); + } + mutex_unlock(&isert_conn->state_mutex); err = 0; } break; default: - pr_err("Illegal event:%d, ignored\n", ev_type); + PRINT_ERROR("Illegal event:%d, ignored", ev_type); break; } if (unlikely(err)) - pr_err("Failed to handle rdma cm evt:%d, err:%d\n", - ev_type, err); + PRINT_ERROR("Failed to handle rdma cm evt:%d, err:%d", + ev_type, err); out: TRACE_EXIT_RES(err); @@ -1660,14 +1762,14 @@ struct isert_portal *isert_portal_create(void) int err; if (unlikely(!try_module_get(THIS_MODULE))) { - pr_err("Unable increment module reference\n"); + PRINT_ERROR("Unable increment module reference"); portal = ERR_PTR(-EINVAL); goto out; } portal = kzalloc(sizeof(*portal), GFP_KERNEL); if (unlikely(!portal)) { - pr_err("Unable to allocate struct portal\n"); + PRINT_ERROR("Unable to allocate struct portal"); portal = ERR_PTR(-ENOMEM); goto err_alloc; } @@ -1684,7 +1786,7 @@ struct isert_portal *isert_portal_create(void) #endif if (unlikely(IS_ERR(cm_id))) { err = PTR_ERR(cm_id); - pr_err("Failed to create rdma id, err:%d\n", err); + PRINT_ERROR("Failed to create rdma id, err:%d", err); goto create_id_err; } portal->cm_id = cm_id; @@ -1696,7 +1798,7 @@ struct isert_portal *isert_portal_create(void) rdma_set_afonly(cm_id, 1); #endif - pr_info("Created iser portal cm_id:%p\n", cm_id); + PRINT_INFO("Created iser portal cm_id:%p", cm_id); out: return portal; @@ -1717,12 +1819,12 @@ int isert_portal_listen(struct isert_portal *portal, TRACE_ENTRY(); err = rdma_bind_addr(portal->cm_id, sa); if (err) { - pr_warn("Failed to bind rdma addr, err:%d\n", err); + PRINT_WARNING("Failed to bind rdma addr, err:%d", err); goto out; } err = rdma_listen(portal->cm_id, ISER_LISTEN_BACKLOG); if (err) { - pr_err("Failed rdma listen, err:%d\n", err); + PRINT_ERROR("Failed rdma listen, err:%d", err); goto out; } memcpy(&portal->addr, sa, addr_len); @@ -1730,33 +1832,33 @@ int isert_portal_listen(struct isert_portal *portal, switch (sa->sa_family) { case AF_INET: #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 33) - pr_info("iser portal cm_id:%p listens on: " - NIPQUAD_FMT ":%d\n", portal->cm_id, - NIPQUAD(((struct sockaddr_in *)sa)->sin_addr.s_addr), - (int)ntohs(((struct sockaddr_in *)sa)->sin_port)); + PRINT_INFO("iser portal cm_id:%p listens on: " + NIPQUAD_FMT ":%d", portal->cm_id, + NIPQUAD(((struct sockaddr_in *)sa)->sin_addr.s_addr), + (int)ntohs(((struct sockaddr_in *)sa)->sin_port)); #else - pr_info("iser portal cm_id:%p listens on: " - "%pI4:%d\n", portal->cm_id, - &((struct sockaddr_in *)sa)->sin_addr.s_addr, - (int)ntohs(((struct sockaddr_in *)sa)->sin_port)); + PRINT_INFO("iser portal cm_id:%p listens on: " + "%pI4:%d", portal->cm_id, + &((struct sockaddr_in *)sa)->sin_addr.s_addr, + (int)ntohs(((struct sockaddr_in *)sa)->sin_port)); #endif break; case AF_INET6: #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29) - pr_info("iser portal cm_id:%p listens on: " - NIP6_FMT " %d\n", - portal->cm_id, - NIP6(((struct sockaddr_in6 *)sa)->sin6_addr), - (int)ntohs(((struct sockaddr_in6 *)sa)->sin6_port)); + PRINT_INFO("iser portal cm_id:%p listens on: " + NIP6_FMT " %d", + portal->cm_id, + NIP6(((struct sockaddr_in6 *)sa)->sin6_addr), + (int)ntohs(((struct sockaddr_in6 *)sa)->sin6_port)); #else - pr_info("iser portal cm_id:%p listens on: " - "%pI6 %d\n", portal->cm_id, - &((struct sockaddr_in6 *)sa)->sin6_addr, - (int)ntohs(((struct sockaddr_in6 *)sa)->sin6_port)); + PRINT_INFO("iser portal cm_id:%p listens on: " + "%pI6 %d", portal->cm_id, + &((struct sockaddr_in6 *)sa)->sin6_addr, + (int)ntohs(((struct sockaddr_in6 *)sa)->sin6_port)); #endif break; default: - pr_err("Unknown address family\n"); + PRINT_ERROR("Unknown address family"); err = -EINVAL; goto out; } @@ -1770,7 +1872,7 @@ static void isert_portal_free(struct isert_portal *portal) { lockdep_assert_held(&dev_list_mutex); - if (!list_empty(&portal->conn_list)) + if (portal->refcnt > 0) return; kfree(portal); @@ -1781,7 +1883,7 @@ void isert_portal_release(struct isert_portal *portal) { struct isert_connection *conn; - pr_info("iser portal cm_id:%p releasing\n", portal->cm_id); + PRINT_INFO("iser portal cm_id:%p releasing", portal->cm_id); if (portal->cm_id) { rdma_destroy_id(portal->cm_id); @@ -1791,11 +1893,20 @@ void isert_portal_release(struct isert_portal *portal) isert_portal_list_remove(portal); mutex_lock(&dev_list_mutex); - list_for_each_entry(conn, &portal->conn_list, portal_node) + list_for_each_entry(conn, &portal->conn_list, portal_node) { isert_conn_disconnect(conn); + if (!test_and_set_bit(ISERT_CONNECTION_CLOSE, &conn->flags)) + isert_sched_conn_closed(conn); + } portal->state = ISERT_PORTAL_INACTIVE; isert_portal_free(portal); mutex_unlock(&dev_list_mutex); + + while (portal->refcnt > 0) { + msleep(100); + } + + PRINT_INFO("done releasing portal %p", portal); } struct isert_portal *isert_portal_start(struct sockaddr *sa, size_t addr_len) diff --git a/iscsi-scst/kernel/isert-scst/isert.c b/iscsi-scst/kernel/isert-scst/isert.c index 2a3693754..4d51073f8 100644 --- a/iscsi-scst/kernel/isert-scst/isert.c +++ b/iscsi-scst/kernel/isert-scst/isert.c @@ -78,6 +78,11 @@ static void isert_mark_conn_closed(struct iscsi_conn *conn, int flags) static void isert_close_conn(struct iscsi_conn *conn, int flags) { + struct isert_conn_dev *dev; + + dev = isert_get_priv(conn); + if (dev) + dev->state = CS_DISCONNECTED; } static int isert_receive_cmnd_data(struct iscsi_cmnd *cmnd) @@ -283,7 +288,7 @@ static void isert_free_conn(struct iscsi_conn *conn) isert_free_connection(conn); } -int isert_handle_close_connection(struct iscsi_conn *conn) +void isert_handle_close_connection(struct iscsi_conn *conn) { isert_mark_conn_closed(conn, 0); /* @@ -295,7 +300,6 @@ int isert_handle_close_connection(struct iscsi_conn *conn) isert_free_connection(conn); else start_close_conn(conn); - return 0; } int isert_pdu_rx(struct iscsi_cmnd *cmnd) @@ -393,8 +397,8 @@ int isert_pdu_sent(struct iscsi_cmnd *pdu) struct iscsi_target *target = pdu->conn->session->target; PRINT_INFO("Closing all connections for target %x at " - "initiator's %s request", target->tid, - conn->session->initiator_name); + "initiator's %s request", target->tid, + conn->session->initiator_name); mutex_lock(&target->target_mutex); target_del_all_sess(target, 0); mutex_unlock(&target->target_mutex); @@ -492,7 +496,7 @@ static int __init isert_init_module(void) int ret; if (isert_nr_devs > 999) { - PRINT_ERROR("Invalid argument for isert_nr_devs provded: %d\n", + PRINT_ERROR("Invalid argument for isert_nr_devs provded: %d", isert_nr_devs); ret = -EINVAL; goto out; diff --git a/iscsi-scst/kernel/isert-scst/isert.h b/iscsi-scst/kernel/isert-scst/isert.h index 0c5113edf..b66845b99 100644 --- a/iscsi-scst/kernel/isert-scst/isert.h +++ b/iscsi-scst/kernel/isert-scst/isert.h @@ -71,7 +71,7 @@ struct isert_listener_dev { struct cdev cdev; dev_t devno; wait_queue_head_t waitqueue; - spinlock_t conn_lock; + struct mutex conn_lock; struct list_head new_conn_list; struct list_head curr_conn_list; struct isert_addr_info info; @@ -129,7 +129,7 @@ int isert_conn_alloc(struct iscsi_session *session, struct iscsi_kern_conn_info *info, struct iscsi_conn **new_conn, struct iscsit_transport *t); -int isert_handle_close_connection(struct iscsi_conn *conn); +void isert_handle_close_connection(struct iscsi_conn *conn); void isert_close_all_portals(void); void isert_del_timer(struct isert_conn_dev *dev); diff --git a/iscsi-scst/kernel/isert-scst/isert_login.c b/iscsi-scst/kernel/isert-scst/isert_login.c index 9d78bb1bc..c43193bc4 100644 --- a/iscsi-scst/kernel/isert-scst/isert_login.c +++ b/iscsi-scst/kernel/isert-scst/isert_login.c @@ -49,8 +49,11 @@ #include "isert_dbg.h" #include "../iscsi.h" #include "isert.h" +#include "iser.h" #include "iser_datamover.h" +static DEFINE_MUTEX(conn_mgmt_mutex); + static unsigned int n_devs; static int isert_major; @@ -67,7 +70,7 @@ static struct isert_conn_dev *get_available_dev(struct isert_listener_dev *dev, unsigned int i; struct isert_conn_dev *res = NULL; - spin_lock(&dev->conn_lock); + mutex_lock(&dev->conn_lock); for (i = 0; i < n_devs; ++i) { if (!isert_conn_devices[i].occupied) { res = &isert_conn_devices[i]; @@ -78,7 +81,7 @@ static struct isert_conn_dev *get_available_dev(struct isert_listener_dev *dev, break; } } - spin_unlock(&dev->conn_lock); + mutex_unlock(&dev->conn_lock); return res; } @@ -107,9 +110,10 @@ static void isert_kref_release_dev(struct kref *kref) static void isert_dev_release(struct isert_conn_dev *dev) { - spin_lock(&isert_listen_dev.conn_lock); + sBUG_ON(atomic_read(&dev->kref.refcount) == 0); + mutex_lock(&isert_listen_dev.conn_lock); kref_put(&dev->kref, isert_kref_release_dev); - spin_unlock(&isert_listen_dev.conn_lock); + mutex_unlock(&isert_listen_dev.conn_lock); } #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) @@ -137,7 +141,7 @@ static void isert_conn_timer_fn(unsigned long arg) conn_dev->timer_active = 0; - PRINT_ERROR("Timeout on connection %p\n", conn_dev->conn); + PRINT_ERROR("Timeout on connection %p", conn_dev->conn); schedule_work(&conn->close_work); @@ -153,7 +157,7 @@ static int add_new_connection(struct isert_listener_dev *dev, TRACE_ENTRY(); if (!conn_dev) { - PRINT_WARNING("%s", "Unable to allocate new connection"); + PRINT_WARNING("Unable to allocate new connection"); res = -ENOSPC; goto out; } @@ -181,9 +185,9 @@ static bool have_new_connection(struct isert_listener_dev *dev) { bool ret; - spin_lock(&dev->conn_lock); + mutex_lock(&dev->conn_lock); ret = !list_empty(&dev->new_conn_list); - spin_unlock(&dev->conn_lock); + mutex_unlock(&dev->conn_lock); return ret; } @@ -203,6 +207,8 @@ int isert_conn_alloc(struct iscsi_session *session, lockdep_assert_held(&session->target->target_mutex); + mutex_lock(&conn_mgmt_mutex); + if (unlikely(!filp)) { res = -EBADF; goto out; @@ -210,6 +216,14 @@ int isert_conn_alloc(struct iscsi_session *session, dev = filp->private_data; + if (unlikely(dev->state == CS_DISCONNECTED)) { + fput(filp); + res = -EBADF; + goto out; + } + + sBUG_ON(dev->state != CS_RSP_FINISHED); + cmnd = dev->login_rsp; sBUG_ON(cmnd == NULL); @@ -265,6 +279,7 @@ cleanup_conn: conn->session = NULL; isert_close_connection(conn); out: + mutex_unlock(&conn_mgmt_mutex); TRACE_EXIT_RES(res); return res; } @@ -304,8 +319,8 @@ static void isert_delete_conn_dev(struct isert_conn_dev *conn_dev) isert_del_timer(conn_dev); if (!test_and_set_bit(ISERT_CONN_PASSED, &conn_dev->flags)) { - BUG_ON(conn_dev->conn == NULL); - isert_close_connection(conn_dev->conn); + if (conn_dev->conn) + isert_close_connection(conn_dev->conn); } } @@ -314,13 +329,13 @@ static int isert_listen_release(struct inode *inode, struct file *filp) struct isert_listener_dev *dev = filp->private_data; struct isert_conn_dev *conn_dev; - spin_lock(&isert_listen_dev.conn_lock); + mutex_lock(&isert_listen_dev.conn_lock); list_for_each_entry(conn_dev, &dev->new_conn_list, conn_list_entry) isert_delete_conn_dev(conn_dev); list_for_each_entry(conn_dev, &dev->curr_conn_list, conn_list_entry) isert_delete_conn_dev(conn_dev); - spin_unlock(&isert_listen_dev.conn_lock); + mutex_unlock(&isert_listen_dev.conn_lock); atomic_inc(&dev->available); return 0; @@ -347,16 +362,16 @@ wait_for_connection: goto out; } - spin_lock(&dev->conn_lock); + mutex_lock(&dev->conn_lock); if (list_empty(&dev->new_conn_list)) { /* could happen if we got disconnect */ - spin_unlock(&dev->conn_lock); + mutex_unlock(&dev->conn_lock); goto wait_for_connection; } conn_dev = list_first_entry(&dev->new_conn_list, struct isert_conn_dev, conn_list_entry); list_move(&conn_dev->conn_list_entry, &dev->curr_conn_list); - spin_unlock(&dev->conn_lock); + mutex_unlock(&dev->conn_lock); to_write = min_t(size_t, sizeof(k_buff), count); res = scnprintf(k_buff, to_write, "/dev/"ISER_CONN_DEV_PREFIX"%d", @@ -385,13 +400,13 @@ static long isert_listen_ioctl(struct file *filp, unsigned int cmd, case SET_LISTEN_ADDR: rc = copy_from_user(&dev->info, ptr, sizeof(dev->info)); if (unlikely(rc != 0)) { - PRINT_ERROR("Failed to copy %d user's bytes\n", rc); + PRINT_ERROR("Failed to copy %d user's bytes", rc); res = -EFAULT; goto out; } if (unlikely(dev->free_portal_idx >= ISERT_MAX_PORTALS)) { - PRINT_ERROR("Maximum number of portals exceeded: %d\n", + PRINT_ERROR("Maximum number of portals exceeded: %d", ISERT_MAX_PORTALS); res = -EINVAL; goto out; @@ -407,7 +422,7 @@ static long isert_listen_ioctl(struct file *filp, unsigned int cmd, portal = isert_portal_add((struct sockaddr *)&dev->info.addr, dev->info.addr_len); if (IS_ERR(portal)) { - PRINT_ERROR("Unable to add portal of size %zu\n", + PRINT_ERROR("Unable to add portal of size %zu", dev->info.addr_len); res = PTR_ERR(portal); goto out; @@ -431,36 +446,61 @@ int isert_conn_established(struct iscsi_conn *iscsi_conn, return add_new_connection(&isert_listen_dev, iscsi_conn); } -int isert_connection_closed(struct iscsi_conn *iscsi_conn) +static void isert_dev_disconnect(struct iscsi_conn* iscsi_conn) { - int res = 0; + struct isert_conn_dev* dev = isert_get_priv(iscsi_conn); + if (dev) { + isert_del_timer(dev); + dev->state = CS_DISCONNECTED; + dev->conn = NULL; + if (dev->login_req) { + isert_task_abort(dev->login_req); + spin_lock(&dev->pdu_lock); + dev->login_req = NULL; + spin_unlock(&dev->pdu_lock); + } + wake_up(&dev->waitqueue); + isert_dev_release(dev); + isert_set_priv(iscsi_conn, NULL); + } +} + +void isert_connection_closed(struct iscsi_conn *iscsi_conn) +{ TRACE_ENTRY(); + mutex_lock(&conn_mgmt_mutex); + if (iscsi_conn->rd_state) { - res = isert_handle_close_connection(iscsi_conn); + mutex_unlock(&conn_mgmt_mutex); + isert_handle_close_connection(iscsi_conn); } else { - struct isert_conn_dev *dev = isert_get_priv(iscsi_conn); - - if (dev) { - isert_del_timer(dev); - dev->state = CS_DISCONNECTED; - if (dev->login_req) { - res = isert_task_abort(dev->login_req); - spin_lock(&dev->pdu_lock); - dev->login_req = NULL; - spin_unlock(&dev->pdu_lock); - } - - wake_up(&dev->waitqueue); - isert_dev_release(dev); - } - + isert_dev_disconnect(iscsi_conn); + mutex_unlock(&conn_mgmt_mutex); isert_free_connection(iscsi_conn); } - TRACE_EXIT_RES(res); - return res; + TRACE_EXIT(); +} + +void isert_connection_abort(struct iscsi_conn *iscsi_conn) +{ + struct isert_connection *isert_conn = (struct isert_connection *)iscsi_conn; + + TRACE_ENTRY(); + + mutex_lock(&conn_mgmt_mutex); + + if (!iscsi_conn->rd_state) { + if (!test_and_set_bit(ISERT_DISCON_CALLED, &isert_conn->flags)) { + isert_dev_disconnect(iscsi_conn); + isert_free_connection(iscsi_conn); + } + } + mutex_unlock(&conn_mgmt_mutex); + + TRACE_EXIT(); } static bool will_read_block(struct isert_conn_dev *dev) @@ -492,13 +532,13 @@ static int isert_open(struct inode *inode, struct file *filp) dev = container_of(inode->i_cdev, struct isert_conn_dev, cdev); - spin_lock(&isert_listen_dev.conn_lock); + mutex_lock(&isert_listen_dev.conn_lock); if (unlikely(dev->occupied == 0)) { - spin_unlock(&isert_listen_dev.conn_lock); + mutex_unlock(&isert_listen_dev.conn_lock); res = -ENODEV; /* already closed */ goto out; } - spin_unlock(&isert_listen_dev.conn_lock); + mutex_unlock(&isert_listen_dev.conn_lock); if (unlikely(!atomic_dec_and_test(&dev->available))) { atomic_inc(&dev->available); @@ -506,9 +546,9 @@ static int isert_open(struct inode *inode, struct file *filp) goto out; } - spin_lock(&isert_listen_dev.conn_lock); + mutex_lock(&isert_listen_dev.conn_lock); kref_get(&dev->kref); - spin_unlock(&isert_listen_dev.conn_lock); + mutex_unlock(&isert_listen_dev.conn_lock); filp->private_data = dev; /* for other methods */ @@ -528,13 +568,7 @@ static int isert_release(struct inode *inode, struct file *filp) dev->sg_virt = NULL; dev->is_discovery = 0; - if (!test_and_set_bit(ISERT_CONN_PASSED, &dev->flags)) { - BUG_ON(dev->conn == NULL); - isert_close_connection(dev->conn); - } - - isert_del_timer(dev); - + isert_delete_conn_dev(dev); isert_dev_release(dev); TRACE_EXIT_RES(res); @@ -562,23 +596,33 @@ static ssize_t isert_read(struct file *filp, char __user *buf, size_t count, struct isert_conn_dev *dev = filp->private_data; size_t to_read; - if (dev->state == CS_DISCONNECTED) + mutex_lock(&conn_mgmt_mutex); + + if (dev->state == CS_DISCONNECTED) { + mutex_unlock(&conn_mgmt_mutex); return -EPIPE; + } if (will_read_block(dev)) { int ret; - if (filp->f_flags & O_NONBLOCK) + if (filp->f_flags & O_NONBLOCK) { + mutex_unlock(&conn_mgmt_mutex); return -EAGAIN; + } ret = wait_event_freezable(dev->waitqueue, !will_read_block(dev)); - if (ret < 0) + if (ret < 0) { + mutex_unlock(&conn_mgmt_mutex); return ret; + } } to_read = min(count, dev->read_len); - if (copy_to_user(buf, dev->read_buf, to_read)) + if (copy_to_user(buf, dev->read_buf, to_read)) { + mutex_unlock(&conn_mgmt_mutex); return -EFAULT; + } dev->read_len -= to_read; dev->read_buf += to_read; @@ -590,8 +634,10 @@ static ssize_t isert_read(struct file *filp, char __user *buf, size_t count, dev->sg_virt = isert_vmap_sg(dev->pages, dev->login_req->sg, dev->login_req->sg_cnt); - if (!dev->sg_virt) + if (!dev->sg_virt) { + mutex_unlock(&conn_mgmt_mutex); return -ENOMEM; + } dev->read_buf = dev->sg_virt + ISER_HDRS_SZ; dev->state = CS_REQ_DATA; } @@ -610,11 +656,12 @@ static ssize_t isert_read(struct file *filp, char __user *buf, size_t count, break; default: - PRINT_ERROR("Invalid state in %s (%d)\n", __func__, - dev->state); + PRINT_ERROR("Invalid state %d", dev->state); to_read = 0; } + mutex_unlock(&conn_mgmt_mutex); + return to_read; } @@ -624,12 +671,18 @@ static ssize_t isert_write(struct file *filp, const char __user *buf, struct isert_conn_dev *dev = filp->private_data; size_t to_write; - if (dev->state == CS_DISCONNECTED) + mutex_lock(&conn_mgmt_mutex); + + if (dev->state == CS_DISCONNECTED) { + mutex_unlock(&conn_mgmt_mutex); return -EPIPE; + } to_write = min(count, dev->write_len); - if (copy_from_user(dev->write_buf, buf, to_write)) + if (copy_from_user(dev->write_buf, buf, to_write)) { + mutex_unlock(&conn_mgmt_mutex); return -EFAULT; + } dev->write_len -= to_write; dev->write_buf += to_write; @@ -641,8 +694,10 @@ static ssize_t isert_write(struct file *filp, const char __user *buf, dev->sg_virt = isert_vmap_sg(dev->pages, dev->login_rsp->sg, dev->login_rsp->sg_cnt); - if (!dev->sg_virt) + if (!dev->sg_virt) { + mutex_unlock(&conn_mgmt_mutex); return -ENOMEM; + } dev->write_buf = dev->sg_virt + ISER_HDRS_SZ; dev->write_len = dev->login_rsp->bufflen - sizeof(dev->login_rsp->pdu.bhs); @@ -654,11 +709,12 @@ static ssize_t isert_write(struct file *filp, const char __user *buf, break; default: - PRINT_ERROR("Invalid state in %s (%d)\n", __func__, - dev->state); + PRINT_ERROR("Invalid state %d", dev->state); to_write = 0; } + mutex_unlock(&conn_mgmt_mutex); + return to_write; } @@ -678,6 +734,8 @@ static long isert_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) TRACE_ENTRY(); + mutex_lock(&conn_mgmt_mutex); + if (dev->state == CS_DISCONNECTED) { res = -EPIPE; goto out; @@ -764,6 +822,7 @@ static long isert_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } out: + mutex_unlock(&conn_mgmt_mutex); TRACE_EXIT_RES(res); return res; } @@ -914,7 +973,7 @@ static void __init isert_setup_listener_cdev(struct isert_listener_dev *dev) init_waitqueue_head(&dev->waitqueue); INIT_LIST_HEAD(&dev->new_conn_list); INIT_LIST_HEAD(&dev->curr_conn_list); - spin_lock_init(&dev->conn_lock); + mutex_init(&dev->conn_lock); atomic_set(&dev->available, 1); err = cdev_add(&dev->cdev, dev->devno, 1); /* Fail gracefully if need be */ @@ -944,7 +1003,7 @@ int __init isert_init_login_devs(unsigned int ndevs) isert_major = MAJOR(devno); if (unlikely(res < 0)) { - PRINT_ERROR("isert: can't get major %d\n", isert_major); + PRINT_ERROR("can't get major %d", isert_major); goto out; } @@ -969,7 +1028,7 @@ int __init isert_init_login_devs(unsigned int ndevs) res = isert_datamover_init(); if (unlikely(res)) { - PRINT_ERROR("Unable to initialize datamover: %d\n", res); + PRINT_ERROR("Unable to initialize datamover: %d", res); goto fail; }