From 6637df7f0c91735148ef49033294ead01c551412 Mon Sep 17 00:00:00 2001 From: Yan Burman Date: Thu, 27 Feb 2014 06:30:26 +0000 Subject: [PATCH] isert: Rework disconnect handling Instead of waiting for TIMEDWAIT event when disconnecting and blocking, start the disconnect process when TIMEDWAIT event is received and only perform the actual rdma_disconnect upon disconnect request. Note that rdma_disconnect can not be called from atomic context, so need to execute it from workqueue. Signed-off-by: Yan Burman git-svn-id: http://svn.code.sf.net/p/scst/svn/branches/iser@5317 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- iscsi-scst/kernel/iscsi.h | 1 + iscsi-scst/kernel/isert-scst/iser.h | 6 +- iscsi-scst/kernel/isert-scst/iser_datamover.c | 3 +- iscsi-scst/kernel/isert-scst/iser_rdma.c | 22 ++----- iscsi-scst/kernel/isert-scst/isert.c | 58 +++--------------- iscsi-scst/kernel/isert-scst/isert.h | 5 -- iscsi-scst/kernel/isert-scst/isert_login.c | 61 ++++++++----------- 7 files changed, 41 insertions(+), 115 deletions(-) diff --git a/iscsi-scst/kernel/iscsi.h b/iscsi-scst/kernel/iscsi.h index ae43e34bd..d93c78d4f 100644 --- a/iscsi-scst/kernel/iscsi.h +++ b/iscsi-scst/kernel/iscsi.h @@ -310,6 +310,7 @@ struct iscsi_conn { #else struct work_struct nop_in_delayed_work; #endif + struct work_struct close_work; unsigned int nop_in_interval; /* in jiffies */ unsigned int nop_in_timeout; /* in jiffies */ struct list_head nop_req_list; diff --git a/iscsi-scst/kernel/isert-scst/iser.h b/iscsi-scst/kernel/isert-scst/iser.h index 727ea4116..a2266a009 100644 --- a/iscsi-scst/kernel/isert-scst/iser.h +++ b/iscsi-scst/kernel/isert-scst/iser.h @@ -102,8 +102,7 @@ struct isert_cq { int idx; }; -#define ISERT_TIMEWAIT_RECEIVED 0 -#define ISERT_CONNECTION_ABORTED 1 +#define ISERT_CONNECTION_ABORTED 0 struct isert_connection { struct iscsi_conn iscsi ____cacheline_aligned; @@ -157,7 +156,6 @@ struct isert_connection { struct list_head dev_node; struct list_head portal_node; - wait_queue_head_t waitQ; unsigned long flags; struct work_struct close_work; struct kref kref; @@ -227,8 +225,8 @@ int isert_post_send(struct isert_connection *isert_conn, int isert_alloc_conn_resources(struct isert_connection *isert_conn); void isert_free_conn_resources(struct isert_connection *isert_conn); -void isert_conn_close(struct isert_connection *isert_conn, int do_flush); void isert_conn_free(struct isert_connection *isert_conn); +void isert_conn_disconnect(struct isert_connection *isert_conn); static inline struct isert_connection *isert_conn_alloc(void) { diff --git a/iscsi-scst/kernel/isert-scst/iser_datamover.c b/iscsi-scst/kernel/isert-scst/iser_datamover.c index c09897466..2dcbf9364 100644 --- a/iscsi-scst/kernel/isert-scst/iser_datamover.c +++ b/iscsi-scst/kernel/isert-scst/iser_datamover.c @@ -272,7 +272,8 @@ int isert_close_connection(struct iscsi_conn *iscsi_conn) { struct isert_connection *isert_conn = (struct isert_connection *)iscsi_conn; - isert_conn_close(isert_conn, 1); + isert_conn_disconnect(isert_conn); + return 0; } diff --git a/iscsi-scst/kernel/isert-scst/iser_rdma.c b/iscsi-scst/kernel/isert-scst/iser_rdma.c index 376264f72..28f164211 100644 --- a/iscsi-scst/kernel/isert-scst/iser_rdma.c +++ b/iscsi-scst/kernel/isert-scst/iser_rdma.c @@ -118,7 +118,7 @@ int isert_post_send(struct isert_connection *isert_conn, return err; } -static void isert_conn_disconnect(struct isert_connection *isert_conn) +void isert_conn_disconnect(struct isert_connection *isert_conn) { int err = rdma_disconnect(isert_conn->cm_id); if (unlikely(err)) @@ -310,7 +310,7 @@ static void isert_recv_completion_handler(struct isert_wr *wr) if (unlikely(err)) { pr_err("err:%d while handling iser pdu\n", err); - isert_conn_close(wr->conn, 0); + isert_conn_disconnect(wr->conn); } TRACE_EXIT(); @@ -1022,8 +1022,6 @@ static struct isert_connection *isert_conn_create(struct rdma_cm_id *cm_id, kref_init(&isert_conn->kref); - init_waitqueue_head(&isert_conn->waitQ); - pr_info("iser created connection cm_id:%p\n", cm_id); TRACE_EXIT(); return isert_conn; @@ -1045,17 +1043,6 @@ fail_get: /* start closing process; * only when all buffers released, can free */ -void isert_conn_close(struct isert_connection *isert_conn, int do_flush) -{ - isert_conn_disconnect(isert_conn); - if (do_flush) { - wait_event_interruptible(isert_conn->waitQ, - test_bit(ISERT_TIMEWAIT_RECEIVED, - &isert_conn->flags)); - flush_workqueue(isert_conn->cq_desc->cq_workqueue); - } -} - static void isert_kref_free(struct kref *kref) { struct isert_connection *isert_conn = container_of(kref, @@ -1068,6 +1055,8 @@ static void isert_kref_free(struct kref *kref) pr_info("isert_conn_free conn:%p\n", isert_conn); + flush_workqueue(isert_conn->cq_desc->cq_workqueue); + isert_free_conn_resources(isert_conn); isert_conn_qp_destroy(isert_conn); @@ -1100,9 +1089,6 @@ static void isert_conn_closed_do_work(struct work_struct *work) struct isert_connection *isert_conn = container_of(work, struct isert_connection, close_work); - set_bit(ISERT_TIMEWAIT_RECEIVED, &isert_conn->flags); - wake_up_interruptible(&isert_conn->waitQ); - /* notify upper layer */ if (!test_bit(ISERT_CONNECTION_ABORTED, &isert_conn->flags)) isert_connection_closed(&isert_conn->iscsi); diff --git a/iscsi-scst/kernel/isert-scst/isert.c b/iscsi-scst/kernel/isert-scst/isert.c index b62a8a6fe..844fba8ff 100644 --- a/iscsi-scst/kernel/isert-scst/isert.c +++ b/iscsi-scst/kernel/isert-scst/isert.c @@ -52,41 +52,8 @@ module_param(isert_nr_devs, uint, S_IRUGO); MODULE_PARM_DESC(isert_nr_devs, "Maximum concurrent number of connection requests to handle."); -static void isert_do_close_conn(struct iscsi_conn *conn) -{ - isert_close_connection(conn); - start_close_conn(conn); -} - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) -static void isert_close_conn_fn(void *ctx) -#else -static void isert_close_conn_fn(struct work_struct *work) -#endif -{ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) - struct isert_close_conn_work *conn_work = ctx; -#else - struct isert_close_conn_work *conn_work = container_of(work, - struct isert_close_conn_work, close_work); -#endif - struct iscsi_conn *conn = conn_work->conn; - - /* Take care of case where our connection is being closed - * without being connected to a session - if connection allocation - * failed for some reason */ - if (unlikely(!conn->session)) - isert_free_connection(conn); - else - isert_do_close_conn(conn); - - kfree(conn_work); -} - static void isert_mark_conn_closed(struct iscsi_conn *conn, int flags) { - struct isert_close_conn_work *conn_work; - TRACE_ENTRY(); if (flags & ISCSI_CONN_ACTIVE_CLOSE) conn->active_close = 1; @@ -97,25 +64,9 @@ static void isert_mark_conn_closed(struct iscsi_conn *conn, int flags) if (!conn->closing) { conn->closing = 1; - - conn_work = kmalloc(sizeof(*conn_work), GFP_ATOMIC); - if (unlikely(!conn_work)) { - PRINT_CRIT_ERROR("Unable to allocate isert_close_conn_work for conn %p\n", - conn); - goto out; - } - - conn_work->conn = conn; -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) - INIT_WORK(&conn_work->close_work, isert_close_conn_fn, - conn_work); -#else - INIT_WORK(&conn_work->close_work, isert_close_conn_fn); -#endif - schedule_work(&conn_work->close_work); + schedule_work(&conn->close_work); } -out: TRACE_EXIT(); } @@ -326,6 +277,13 @@ static void isert_conn_free(struct iscsi_conn *conn) int isert_handle_close_connection(struct iscsi_conn *conn) { isert_mark_conn_closed(conn, 0); + /* Take care of case where our connection is being closed + * without being connected to a session - if connection allocation + * failed for some reason */ + if (unlikely(!conn->session)) + isert_free_connection(conn); + else + start_close_conn(conn); return 0; } diff --git a/iscsi-scst/kernel/isert-scst/isert.h b/iscsi-scst/kernel/isert-scst/isert.h index 1e692fd01..78223527b 100644 --- a/iscsi-scst/kernel/isert-scst/isert.h +++ b/iscsi-scst/kernel/isert-scst/isert.h @@ -81,11 +81,6 @@ struct isert_listener_dev { int free_portal_idx; }; -struct isert_close_conn_work { - struct work_struct close_work; - struct iscsi_conn *conn; -}; - enum isert_conn_dev_state { CS_INIT, CS_REQ_BHS, diff --git a/iscsi-scst/kernel/isert-scst/isert_login.c b/iscsi-scst/kernel/isert-scst/isert_login.c index 9a96ade4f..0d6ea579b 100644 --- a/iscsi-scst/kernel/isert-scst/isert_login.c +++ b/iscsi-scst/kernel/isert-scst/isert_login.c @@ -35,7 +35,6 @@ #include #include -#include /* kmalloc() */ #include /* everything... */ #include /* error codes */ #include @@ -103,29 +102,10 @@ static void release_dev(struct isert_conn_dev *dev) spin_unlock(&isert_listen_dev.conn_lock); } -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) -static void isert_login_close_conn_fn(void *ctx) -#else -static void isert_login_close_conn_fn(struct work_struct *work) -#endif -{ -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) - struct isert_close_conn_work *conn_work = ctx; -#else - struct isert_close_conn_work *conn_work = container_of(work, - struct isert_close_conn_work, close_work); -#endif - struct iscsi_conn *conn = conn_work->conn; - - isert_close_connection(conn); - - kfree(conn_work); -} - static void isert_conn_timer_fn(unsigned long arg) { struct isert_conn_dev *conn_dev = (struct isert_conn_dev *)arg; - struct isert_close_conn_work *conn_work; + struct iscsi_conn *conn = conn_dev->conn; TRACE_ENTRY(); @@ -133,23 +113,8 @@ static void isert_conn_timer_fn(unsigned long arg) PRINT_ERROR("Timeout on connection %p\n", conn_dev->conn); - conn_work = kmalloc(sizeof(*conn_work), GFP_ATOMIC); - if (unlikely(!conn_work)) { - PRINT_CRIT_ERROR("Unable to allocate isert_close_conn_work for conn %p\n", - conn_dev->conn); - goto out; - } + schedule_work(&conn->close_work); - conn_work->conn = conn_dev->conn; -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) - INIT_WORK(&conn_work->close_work, isert_login_close_conn_fn, - conn_work); -#else - INIT_WORK(&conn_work->close_work, isert_login_close_conn_fn); -#endif - schedule_work(&conn_work->close_work); - -out: TRACE_EXIT(); } @@ -191,6 +156,22 @@ static bool have_new_connection(struct isert_listener_dev *dev) return ret; } +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) +static void isert_close_conn_fn(void *ctx) +#else +static void isert_close_conn_fn(struct work_struct *work) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) + struct iscsi_conn *conn = ctx; +#else + struct iscsi_conn *conn = container_of(work, + struct iscsi_conn, close_work); +#endif + + isert_close_connection(conn); +} + int isert_conn_alloc(struct iscsi_session *session, struct iscsi_kern_conn_info *info, struct iscsi_conn **new_conn, @@ -236,6 +217,12 @@ int isert_conn_alloc(struct iscsi_session *session, conn->transport = t; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) + INIT_WORK(&conn->close_work, isert_close_conn_fn, conn); +#else + INIT_WORK(&conn->close_work, isert_close_conn_fn); +#endif + res = iscsi_init_conn(session, info, conn); if (unlikely(res)) goto cleanup_conn;