From e5c4ac05f6655b6f6507ebee9de254938f35ee4d Mon Sep 17 00:00:00 2001 From: Vladislav Bolkhovitin Date: Wed, 21 Feb 2007 11:43:22 +0000 Subject: [PATCH] A major locking and general code cleanup git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@90 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- scst/include/scsi_tgt.h | 122 ++- scst/src/dev_handlers/scst_fileio.c | 11 +- scst/src/scst.c | 290 ++++--- scst/src/scst_lib.c | 404 +++++----- scst/src/scst_priv.h | 78 +- scst/src/scst_proc.c | 26 +- scst/src/scst_targ.c | 1110 +++++++++++++++------------ 7 files changed, 1177 insertions(+), 864 deletions(-) diff --git a/scst/include/scsi_tgt.h b/scst/include/scsi_tgt.h index b4a997e04..d4922d658 100644 --- a/scst/include/scsi_tgt.h +++ b/scst/include/scsi_tgt.h @@ -150,12 +150,6 @@ /* Thread context required for cmd's processing */ #define SCST_CONTEXT_THREAD 3 -/* - * Additional bit to set processible environment. - * Private for SCST, ie must not be used target drivers. - */ -#define SCST_PROCESSIBLE_ENV 0x10000000 - /************************************************************* ** Values for status parameter of scst_rx_data() *************************************************************/ @@ -365,24 +359,30 @@ */ #define SCST_CMD_XMITTING 4 -/* Set if the cmd was done or aborted out of its SN */ -#define SCST_CMD_OUT_OF_SN 5 - /* Set if the cmd is dead and can be destroyed at any time */ -#define SCST_CMD_CAN_BE_DESTROYED 6 +#define SCST_CMD_CAN_BE_DESTROYED 5 /************************************************************* - ** Tgt_dev's flags + ** Tgt_dev's flags (tgt_dev_flags) *************************************************************/ /* Set if tgt_dev has Unit Attention sense */ -#define SCST_TGT_DEV_UA_PENDING 0 +#define SCST_TGT_DEV_UA_PENDING 0 /* Set if tgt_dev is RESERVED by another session */ -#define SCST_TGT_DEV_RESERVED 1 +#define SCST_TGT_DEV_RESERVED 1 + +/* Set if the corresponding context is atomic */ +#define SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC 5 +#define SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC 6 +#define SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC 7 +#define SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC 8 +#define SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC 9 +#define SCST_TGT_DEV_AFTER_EXEC_ATOMIC 10 + #ifdef DEBUG_TM -#define SCST_TGT_DEV_UNDER_TM_DBG 10 +#define SCST_TGT_DEV_UNDER_TM_DBG 20 #endif /************************************************************* @@ -657,9 +657,7 @@ struct scst_tgt_template * level driver. No return value expected. * This function is expected to be NON-BLOCKING * - * Pay attention to "atomic" attribute of the cmd, which can be get - * by scst_cmd_atomic(): it is true if the function called in the - * atomic (non-sleeping) context. + * Called without any locks held from a thread context. * * MUST HAVE if the target supports ABORTs */ @@ -838,7 +836,7 @@ struct scst_dev_type * - SCST_DEV_TM_NOT_COMPLETED - regular standard actions for the command * should be done * - * Called with BH off. Might be called under a lock and IRQ off. + * Called without any locks held from a thread context. */ int (*task_mgmt_fn) (struct scst_mgmt_cmd *mgmt_cmd, struct scst_tgt_dev *tgt_dev); @@ -907,19 +905,13 @@ struct scst_session atomic_t refcnt; /* get/put counter */ - /* Alive commands for this session. Serialized by scst_list_lock */ - int sess_cmd_count; - /************************************************************* - ** Session's flags. Serialized by scst_list_lock + ** Session's flags. Serialized by scst_mgmt_lock *************************************************************/ /* Set if the session is shutting down */ unsigned int shutting_down:1; - /* Set if the session is waiting in suspended state */ - unsigned int waiting:1; - /**************************************************************/ /* @@ -937,28 +929,22 @@ struct scst_session /* Used for storage of target driver private stuff */ void *tgt_priv; + /* Alive commands for this session, protected by sess_list_lock */ + int sess_cmd_count; + + spinlock_t sess_list_lock; /* protects search_cmd_list, etc */ + /* * List of cmds in this session. Used to find a cmd in the - * session. Protected by scst_list_lock. - * - * ToDo: make it protected by own lock. + * session. Protected by sess_list_lock. */ struct list_head search_cmd_list; struct scst_tgt *tgt; /* corresponding target */ - /* - * List entry for the list that keeps all sessions, which were - * stopped due to device block - */ - struct list_head dev_wait_sess_list_entry; - /* Name of attached initiator */ const char *initiator_name; - /* Used if scst_unregister_session() called in wait mode */ - struct completion *shutdown_compl; - /* List entry of sessions per target */ struct list_head sess_list_entry; @@ -967,11 +953,14 @@ struct scst_session /* * Lists of deffered during session initialization commands. - * Protected by scst_list_lock. + * Protected by sess_list_lock. */ struct list_head init_deferred_cmd_list; struct list_head init_deferred_mcmd_list; + /* Used if scst_unregister_session() called in wait mode */ + struct completion *shutdown_compl; + /* * Functions and data for user callbacks from scst_register_session() * and scst_unregister_session() @@ -991,11 +980,24 @@ enum scst_cmd_queue_type SCST_CMD_QUEUE_ACA }; +struct scst_cmd_lists +{ + spinlock_t cmd_list_lock; + struct list_head active_cmd_list; + wait_queue_head_t cmd_list_waitQ; + struct list_head lists_list_entry; +}; + struct scst_cmd { - /* List entry for global *_cmd_list */ + /* List entry for below *_cmd_lists */ struct list_head cmd_list_entry; + /* Pointer to lists of commands with the lock */ + struct scst_cmd_lists *cmd_lists; + + atomic_t cmd_ref; + struct scst_session *sess; /* corresponding session */ /* Cmd state, one of SCST_CMD_STATE_* constants */ @@ -1019,9 +1021,6 @@ struct scst_cmd /* Set if cmd is being processed in atomic context */ unsigned int atomic:1; - /* Set if cmd must be processed only in non-atomic context */ - unsigned int non_atomic_only:1; - /* Set if cmd is internally generated */ unsigned int internal:1; @@ -1047,14 +1046,6 @@ struct scst_cmd /* Set if the target driver called scst_set_expected() */ unsigned int expected_values_set:1; - /* - * Set if the cmd is being processed from the thread/tasklet, - * i.e. if another cmd is moved to the active list during processing - * of the current one, then another cmd will be processed after - * the current. Helps to save some context switches. - */ - unsigned int processible_env:1; - /* * Set if the cmd was delayed by task management debugging code. * Used only if DEBUG_TM is on. @@ -1100,9 +1091,12 @@ struct scst_cmd */ unsigned int may_need_dma_sync:1; + /* Set if the cmd was done or aborted out of its SN */ + unsigned long out_of_sn:1; + /**************************************************************/ - unsigned long cmd_flags; /* cmd's async flags */ + unsigned long cmd_flags; /* cmd's async flags */ struct scst_tgt_template *tgtt; /* to save extra dereferences */ struct scst_tgt *tgt; /* to save extra dereferences */ @@ -1190,7 +1184,7 @@ struct scst_cmd uint8_t sense_buffer[SCSI_SENSE_BUFFERSIZE]; /* sense buffer */ - /* The corresponding mgmt cmd, if any. Protected by scst_list_lock */ + /* The corresponding mgmt cmd, if any, protected by sess_list_lock */ struct scst_mgmt_cmd *mgmt_cmnd; /* List entry for dev's blocked_cmd_list */ @@ -1227,11 +1221,15 @@ struct scst_mgmt_cmd int fn; unsigned int completed:1; /* set, if the mcmd is completed */ + unsigned int active:1; /* set, if the mcmd is active */ - /* Number of commands to complete before sending response */ + /* + * Number of commands to complete before sending response, + * protected by scst_mcmd_lock + */ int cmd_wait_count; - /* Number of completed commands */ + /* Number of completed commands, protected by scst_mcmd_lock */ int completed_cmd_count; lun_t lun; /* LUN for this mgmt cmd */ @@ -1334,11 +1332,8 @@ struct scst_tgt_dev struct scst_acg_dev *acg_dev; /* corresponding acg_dev */ - /* - * How many cmds alive on this dev in this session. - * Protected by scst_list_lock. - */ - int cmd_count; + /* How many cmds alive on this dev in this session */ + atomic_t cmd_count; spinlock_t tgt_dev_lock; /* per-session device lock */ @@ -1350,16 +1345,19 @@ struct scst_tgt_dev /* Used for storage of dev handler private stuff */ void *dh_priv; + /* Pointer to lists of commands with the lock */ + struct scst_cmd_lists *p_cmd_lists; + /* * Used to execute cmd's in order of arrival. * - * Protected by sn_lock, except next_sn and expected_sn. + * Protected by sn_lock, except curr_sn and expected_sn. * Expected_sn protected by itself, since only one thread, which * processes SN matching command, can increment it at any time. - * Next_sn must be the same type as expected_sn to overflow + * Next_sn must have the same size as expected_sn to overflow * simultaneously. */ - int next_sn; /* protected by scst_list_lock */ + atomic_t curr_sn; int expected_sn; spinlock_t sn_lock; int def_cmd_count; diff --git a/scst/src/dev_handlers/scst_fileio.c b/scst/src/dev_handlers/scst_fileio.c index 9b875284d..251cc1985 100644 --- a/scst/src/dev_handlers/scst_fileio.c +++ b/scst/src/dev_handlers/scst_fileio.c @@ -2325,7 +2325,7 @@ out: return; } -/* Called with BH off. Might be called under lock and IRQ off */ +/* No locks supposed to be held, thread context */ static int fileio_task_mgmt_fn(struct scst_mgmt_cmd *mcmd, struct scst_tgt_dev *tgt_dev) { @@ -2338,12 +2338,7 @@ static int fileio_task_mgmt_fn(struct scst_mgmt_cmd *mcmd, struct scst_fileio_tgt_dev *ftgt_dev = (struct scst_fileio_tgt_dev *)cmd_to_abort->tgt_dev->dh_priv; - /* - * It is safe relating to scst_list_lock despite of lockdep's - * warning. Just don't know how to tell it to lockdep. - */ - /* BH already off */ - spin_lock(&ftgt_dev->fdev_lock); + spin_lock_bh(&ftgt_dev->fdev_lock); if (cmd_to_abort->fileio_in_list) { TRACE(TRACE_MGMT, "Aborting cmd %p and moving it to " "the queue head", cmd_to_abort); @@ -2352,7 +2347,7 @@ static int fileio_task_mgmt_fn(struct scst_mgmt_cmd *mcmd, &ftgt_dev->fdev_cmd_list); wake_up(&ftgt_dev->fdev_waitQ); } - spin_unlock(&ftgt_dev->fdev_lock); + spin_unlock_bh(&ftgt_dev->fdev_lock); } TRACE_EXIT_RES(res); diff --git a/scst/src/scst.c b/scst/src/scst.c index 725e470dc..d8c6fec19 100644 --- a/scst/src/scst.c +++ b/scst/src/scst.c @@ -32,23 +32,18 @@ #include "scst_priv.h" #include "scst_mem.h" -#if defined(DEBUG) || defined(TRACING) -unsigned long scst_trace_flag = SCST_DEFAULT_LOG_FLAGS; -#endif - /* * All targets, devices and dev_types management is done under * this mutex. */ DECLARE_MUTEX(scst_mutex); -DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ); -LIST_HEAD(scst_dev_wait_sess_list); - LIST_HEAD(scst_template_list); LIST_HEAD(scst_dev_list); LIST_HEAD(scst_dev_type_list); +spinlock_t scst_main_lock = SPIN_LOCK_UNLOCKED; + struct kmem_cache *scst_mgmt_cachep; mempool_t *scst_mgmt_mempool; struct kmem_cache *scst_ua_cachep; @@ -60,32 +55,37 @@ struct kmem_cache *scst_acgd_cachep; LIST_HEAD(scst_acg_list); struct scst_acg *scst_default_acg; +spinlock_t scst_init_lock = SPIN_LOCK_UNLOCKED; +DECLARE_WAIT_QUEUE_HEAD(scst_init_cmd_list_waitQ); +LIST_HEAD(scst_init_cmd_list); +unsigned int scst_init_poll_cnt; + struct kmem_cache *scst_cmd_cachep; +#if defined(DEBUG) || defined(TRACING) +unsigned long scst_trace_flag = SCST_DEFAULT_LOG_FLAGS; +#endif + unsigned long scst_flags; atomic_t scst_cmd_count = ATOMIC_INIT(0); -spinlock_t scst_list_lock = SPIN_LOCK_UNLOCKED; -LIST_HEAD(scst_active_cmd_list); -LIST_HEAD(scst_init_cmd_list); -LIST_HEAD(scst_cmd_list); -DECLARE_WAIT_QUEUE_HEAD(scst_list_waitQ); spinlock_t scst_cmd_mem_lock = SPIN_LOCK_UNLOCKED; unsigned long scst_cur_cmd_mem, scst_cur_max_cmd_mem; - -struct tasklet_struct scst_tasklets[NR_CPUS]; +unsigned long scst_max_cmd_mem; struct scst_sgv_pools scst_sgv; +struct scst_cmd_lists scst_main_cmd_lists; + +struct scst_tasklet scst_tasklets[NR_CPUS]; + #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) DECLARE_WORK(scst_cmd_mem_work, scst_cmd_mem_work_fn, 0); #else DECLARE_DELAYED_WORK(scst_cmd_mem_work, scst_cmd_mem_work_fn); #endif -unsigned long scst_max_cmd_mem; - -LIST_HEAD(scst_mgmt_cmd_list); +spinlock_t scst_mcmd_lock = SPIN_LOCK_UNLOCKED; LIST_HEAD(scst_active_mgmt_cmd_list); LIST_HEAD(scst_delayed_mgmt_cmd_list); DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_cmd_list_waitQ); @@ -94,6 +94,11 @@ DECLARE_WAIT_QUEUE_HEAD(scst_mgmt_waitQ); spinlock_t scst_mgmt_lock = SPIN_LOCK_UNLOCKED; LIST_HEAD(scst_sess_mgmt_list); +DECLARE_WAIT_QUEUE_HEAD(scst_dev_cmd_waitQ); + +DECLARE_MUTEX(scst_suspend_mutex);; +LIST_HEAD(scst_cmd_lists_list); /* protected by scst_suspend_mutex */ + static int scst_threads; struct scst_threads_info_t scst_threads_info; @@ -152,6 +157,9 @@ int scst_register_target_template(struct scst_tgt_template *vtt) goto out_err; } + if (vtt->preprocessing_done == NULL) + vtt->preprocessing_done_atomic = 1; + if (down_interruptible(&m) != 0) goto out_err; @@ -265,6 +273,7 @@ struct scst_tgt *scst_register(struct scst_tgt_template *vtt) tgt->retry_timer.data = (unsigned long)tgt; tgt->retry_timer.function = scst_tgt_retry_timer_fn; + scst_suspend_activity(); down(&scst_mutex); if (scst_build_proc_target_entries(tgt) < 0) { @@ -275,6 +284,7 @@ struct scst_tgt *scst_register(struct scst_tgt_template *vtt) list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list); up(&scst_mutex); + scst_resume_activity(); PRINT_INFO_PR("Target for template %s registered successfully", vtt->name); @@ -285,6 +295,7 @@ out: out_up: up(&scst_mutex); + scst_resume_activity(); out_err: PRINT_ERROR_PR("Failed to register target for template %s", vtt->name); @@ -321,12 +332,16 @@ void scst_unregister(struct scst_tgt *tgt) wait_event(tgt->unreg_waitQ, test_sess_list(tgt)); TRACE_DBG("%s", "wait_event() returned"); + scst_suspend_activity(); down(&scst_mutex); + list_del(&tgt->tgt_list_entry); - up(&scst_mutex); scst_cleanup_proc_target_entries(tgt); + up(&scst_mutex); + scst_resume_activity(); + del_timer_sync(&tgt->retry_timer); kfree(tgt); @@ -338,72 +353,81 @@ void scst_unregister(struct scst_tgt *tgt) return; } -/* scst_mutex supposed to be held */ -void __scst_suspend_activity(void) +void scst_suspend_activity(void) { TRACE_ENTRY(); + down(&scst_suspend_mutex); + + TRACE_MGMT_DBG("suspend_count %d", suspend_count); suspend_count++; if (suspend_count > 1) - goto out; + goto out_up; + set_bit(SCST_FLAG_SUSPENDING, &scst_flags); set_bit(SCST_FLAG_SUSPENDED, &scst_flags); smp_mb__after_set_bit(); - TRACE_DBG("Waiting for all %d active commands to complete", + TRACE_MGMT_DBG("Waiting for %d active commands to complete", atomic_read(&scst_cmd_count)); wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0); - TRACE_DBG("%s", "wait_event() returned"); + TRACE_MGMT_DBG("%s", "wait_event() returned"); -out: - TRACE_EXIT(); - return; -} - -void scst_suspend_activity(void) -{ - down(&scst_mutex); - __scst_suspend_activity(); -} - -/* scst_mutex supposed to be held */ -void __scst_resume_activity(void) -{ - struct scst_session *sess, *tsess; - - TRACE_ENTRY(); - - suspend_count--; - if (suspend_count > 0) - goto out; - - clear_bit(SCST_FLAG_SUSPENDED, &scst_flags); + clear_bit(SCST_FLAG_SUSPENDING, &scst_flags); smp_mb__after_clear_bit(); - spin_lock_irq(&scst_list_lock); - list_for_each_entry_safe(sess, tsess, &scst_dev_wait_sess_list, - dev_wait_sess_list_entry) - { - sess->waiting = 0; - list_del(&sess->dev_wait_sess_list_entry); - } - spin_unlock_irq(&scst_list_lock); + TRACE_MGMT_DBG("Waiting for %d active commands finally to complete", + atomic_read(&scst_cmd_count)); + wait_event(scst_dev_cmd_waitQ, atomic_read(&scst_cmd_count) == 0); + TRACE_MGMT_DBG("%s", "wait_event() returned"); - wake_up_all(&scst_list_waitQ); - wake_up_all(&scst_mgmt_cmd_list_waitQ); +out_up: + up(&scst_suspend_mutex); -out: TRACE_EXIT(); return; } void scst_resume_activity(void) { - __scst_resume_activity(); - up(&scst_mutex); + struct scst_cmd_lists *l; + + TRACE_ENTRY(); + + down(&scst_suspend_mutex); + + TRACE_MGMT_DBG("suspend_count %d", suspend_count); + suspend_count--; + if (suspend_count > 0) + goto out_up; + + clear_bit(SCST_FLAG_SUSPENDED, &scst_flags); + smp_mb__after_clear_bit(); + + list_for_each_entry(l, &scst_cmd_lists_list, lists_list_entry) { + wake_up_all(&l->cmd_list_waitQ); + } + wake_up_all(&scst_init_cmd_list_waitQ); + + spin_lock_irq(&scst_mcmd_lock); + if (!list_empty(&scst_delayed_mgmt_cmd_list)) { + struct scst_mgmt_cmd *m; + m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m), + mgmt_cmd_list_entry); + TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active " + "mgmt cmd list", m); + list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list); + } + spin_unlock_irq(&scst_mcmd_lock); + wake_up_all(&scst_mgmt_cmd_list_waitQ); + +out_up: + up(&scst_suspend_mutex); + + TRACE_EXIT(); + return; } -/* Called under scst_mutex */ static int scst_register_device(struct scsi_device *scsidp) { int res = 0; @@ -412,12 +436,15 @@ static int scst_register_device(struct scsi_device *scsidp) TRACE_ENTRY(); + scst_suspend_activity(); + down(&scst_mutex); + dev = scst_alloc_device(GFP_KERNEL); if (dev == NULL) { res = -ENOMEM; - goto out; + goto out_up; } - + dev->rq_disk = alloc_disk(1); if (dev->rq_disk == NULL) { res = -ENOMEM; @@ -438,7 +465,10 @@ static int scst_register_device(struct scsi_device *scsidp) } } -out: +out_up: + up(&scst_mutex); + scst_resume_activity(); + if (res == 0) { PRINT_INFO_PR("Attached SCSI target mid-level at " "scsi%d, channel %d, id %d, lun %d, type %d", @@ -461,10 +491,9 @@ out_free: out_free_dev: scst_free_device(dev); - goto out; + goto out_up; } -/* Called under scst_mutex */ static void scst_unregister_device(struct scsi_device *scsidp) { struct scst_device *d, *dev = NULL; @@ -472,7 +501,8 @@ static void scst_unregister_device(struct scsi_device *scsidp) TRACE_ENTRY(); - __scst_suspend_activity(); + scst_suspend_activity(); + down(&scst_mutex); list_for_each_entry(d, &scst_dev_list, dev_list_entry) { if (d->scsi_dev == scsidp) { @@ -504,16 +534,39 @@ static void scst_unregister_device(struct scsi_device *scsidp) scsidp->channel, scsidp->id, scsidp->lun, scsidp->type); out_unblock: - __scst_resume_activity(); + up(&scst_mutex); + scst_resume_activity(); TRACE_EXIT(); return; } +static int scst_dev_handler_check(struct scst_dev_type *dev_handler) +{ + int res = 0; + + if (dev_handler->parse == NULL) { + PRINT_ERROR_PR("scst dev_type driver %s doesn't have a " + "parse() method.", dev_handler->name); + res = -EINVAL; + goto out; + } + + if (dev_handler->exec == NULL) + dev_handler->exec_atomic = 1; + + if (dev_handler->dev_done == NULL) + dev_handler->dev_done_atomic = 1; + +out: + TRACE_EXIT_RES(res); + return res; +} + int scst_register_virtual_device(struct scst_dev_type *dev_handler, const char *dev_name) { - int res = -EINVAL, rc; + int res, rc; struct scst_device *dev = NULL; TRACE_ENTRY(); @@ -531,6 +584,10 @@ int scst_register_virtual_device(struct scst_dev_type *dev_handler, goto out; } + res = scst_dev_handler_check(dev_handler); + if (res != 0) + goto out; + dev = scst_alloc_device(GFP_KERNEL); if (dev == NULL) { res = -ENOMEM; @@ -540,6 +597,8 @@ int scst_register_virtual_device(struct scst_dev_type *dev_handler, dev->scsi_dev = NULL; dev->virt_name = dev_name; + scst_suspend_activity(); + if (down_interruptible(&scst_mutex) != 0) { res = -EINTR; goto out_free_dev; @@ -559,6 +618,9 @@ int scst_register_virtual_device(struct scst_dev_type *dev_handler, up(&scst_mutex); +out_resume: + scst_resume_activity(); + out: if (res > 0) { PRINT_INFO_PR("Attached SCSI target mid-level to virtual " @@ -578,7 +640,7 @@ out_free_del: out_free_dev: scst_free_device(dev); - goto out; + goto out_resume; } void scst_unregister_virtual_device(int id) @@ -588,10 +650,9 @@ void scst_unregister_virtual_device(int id) TRACE_ENTRY(); + scst_suspend_activity(); down(&scst_mutex); - __scst_suspend_activity(); - list_for_each_entry(d, &scst_dev_list, dev_list_entry) { if (d->virt_id == id) { dev = d; @@ -620,9 +681,8 @@ void scst_unregister_virtual_device(int id) scst_free_device(dev); out_unblock: - __scst_resume_activity(); - up(&scst_mutex); + scst_resume_activity(); TRACE_EXIT(); return; @@ -632,17 +692,14 @@ int scst_register_dev_driver(struct scst_dev_type *dev_type) { struct scst_dev_type *dt; struct scst_device *dev; - int res = 0; + int res; int exist; TRACE_ENTRY(); - if (dev_type->parse == NULL) { - PRINT_ERROR_PR("scst dev_type driver %s doesn't have a " - "parse() method.", dev_type->name); - res = -EINVAL; + res = scst_dev_handler_check(dev_type); + if (res != 0) goto out_err; - } #ifdef FILEIO_ONLY if (dev_type->exec == NULL) { @@ -654,6 +711,8 @@ int scst_register_dev_driver(struct scst_dev_type *dev_type) } #endif + scst_suspend_activity(); + if (down_interruptible(&scst_mutex) != 0) { res = -EINTR; goto out_err; @@ -678,16 +737,15 @@ int scst_register_dev_driver(struct scst_dev_type *dev_type) list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list); - __scst_suspend_activity(); list_for_each_entry(dev, &scst_dev_list, dev_list_entry) { if ((dev->scsi_dev == NULL) || (dev->handler != NULL)) continue; if (dev->scsi_dev->type == dev_type->type) scst_assign_dev_handler(dev, dev_type); } - __scst_resume_activity(); up(&scst_mutex); + scst_resume_activity(); if (res == 0) { PRINT_INFO_PR("Device handler %s for type %d registered " @@ -702,6 +760,7 @@ out_up: up(&scst_mutex); out_err: + scst_resume_activity(); PRINT_ERROR_PR("Failed to register device handler %s for type %d", dev_type->name, dev_type->type); goto out; @@ -715,6 +774,7 @@ void scst_unregister_dev_driver(struct scst_dev_type *dev_type) TRACE_ENTRY(); + scst_suspend_activity(); down(&scst_mutex); list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) { @@ -729,18 +789,17 @@ void scst_unregister_dev_driver(struct scst_dev_type *dev_type) goto out_up; } - __scst_suspend_activity(); list_for_each_entry(dev, &scst_dev_list, dev_list_entry) { if (dev->handler == dev_type) { scst_assign_dev_handler(dev, NULL); TRACE_DBG("Dev handler removed from device %p", dev); } } - __scst_resume_activity(); list_del(&dev_type->dev_type_list_entry); up(&scst_mutex); + scst_resume_activity(); scst_cleanup_proc_dev_handler_dir_entries(dev_type); @@ -753,21 +812,19 @@ out: out_up: up(&scst_mutex); + scst_resume_activity(); goto out; } int scst_register_virtual_dev_driver(struct scst_dev_type *dev_type) { - int res = 0; + int res; TRACE_ENTRY(); - if (dev_type->parse == NULL) { - PRINT_ERROR_PR("scst dev_type driver %s doesn't have a " - "parse() method.", dev_type->name); - res = -EINVAL; + res = scst_dev_handler_check(dev_type); + if (res != 0) goto out_err; - } res = scst_build_proc_dev_handler_dir_entries(dev_type); if (res < 0) @@ -799,7 +856,7 @@ void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type) return; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ int scst_assign_dev_handler(struct scst_device *dev, struct scst_dev_type *handler) { @@ -812,8 +869,6 @@ int scst_assign_dev_handler(struct scst_device *dev, if (dev->handler == handler) goto out; - __scst_suspend_activity(); - if (dev->handler && dev->handler->detach_tgt) { list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, dev_tgt_dev_list_entry) @@ -841,7 +896,7 @@ int scst_assign_dev_handler(struct scst_device *dev, PRINT_ERROR_PR("New device handler's %s attach() " "failed: %d", handler->name, res); } - goto out_resume; + goto out_null; } if (handler && handler->attach_tgt) { @@ -862,10 +917,9 @@ int scst_assign_dev_handler(struct scst_device *dev, } } -out_resume: +out_null: if (res != 0) dev->handler = NULL; - __scst_resume_activity(); out: TRACE_EXIT_RES(res); @@ -885,9 +939,9 @@ out_err_detach_tgt: if (handler && handler->detach) { TRACE_DBG("%s", "Calling handler's detach()"); handler->detach(dev); - TRACE_DBG("%s", "Hhandler's detach() returned"); + TRACE_DBG("%s", "Handler's detach() returned"); } - goto out_resume; + goto out_null; } int scst_cmd_threads_count(void) @@ -959,7 +1013,8 @@ int __scst_add_cmd_threads(int num) PRINT_ERROR_PR("fail to allocate thr %d", res); goto out_error; } - thr->cmd_thread = kthread_run(scst_cmd_thread, 0, "scsi_tgt%d", + thr->cmd_thread = kthread_run(scst_cmd_thread, + &scst_main_cmd_lists, "scsi_tgt%d", scst_thread_num++); if (IS_ERR(thr->cmd_thread)) { res = PTR_ERR(thr->cmd_thread); @@ -1019,6 +1074,8 @@ static void scst_stop_all_threads(void) kthread_stop(scst_threads_info.mgmt_cmd_thread); if (scst_threads_info.mgmt_thread) kthread_stop(scst_threads_info.mgmt_thread); + if (scst_threads_info.init_cmd_thread) + kthread_stop(scst_threads_info.init_cmd_thread); up(&scst_threads_info.cmd_threads_mutex); TRACE_EXIT(); @@ -1036,6 +1093,15 @@ static int scst_start_all_threads(int num) if (res < 0) goto out; + scst_threads_info.init_cmd_thread = kthread_run(scst_init_cmd_thread, + NULL, "scsi_tgt_init"); + if (IS_ERR(scst_threads_info.init_cmd_thread)) { + res = PTR_ERR(scst_threads_info.init_cmd_thread); + PRINT_ERROR_PR("kthread_create() for init cmd failed: %d", res); + scst_threads_info.init_cmd_thread = NULL; + goto out; + } + scst_threads_info.mgmt_cmd_thread = kthread_run(scst_mgmt_cmd_thread, NULL, "scsi_tgt_mc"); if (IS_ERR(scst_threads_info.mgmt_cmd_thread)) { @@ -1062,12 +1128,12 @@ out: void scst_get(void) { - scst_inc_cmd_count(); + __scst_get(0); } void scst_put(void) { - scst_dec_cmd_count(); + __scst_put(); } #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) @@ -1082,10 +1148,7 @@ static int scst_add(struct class_device *cdev, struct class_interface *intf) TRACE_ENTRY(); scsidp = to_scsi_device(cdev->dev); - - down(&scst_mutex); res = scst_register_device(scsidp); - up(&scst_mutex); TRACE_EXIT(); return res; @@ -1102,10 +1165,7 @@ static void scst_remove(struct class_device *cdev, struct class_interface *intf) TRACE_ENTRY(); scsidp = to_scsi_device(cdev->dev); - - down(&scst_mutex); scst_unregister_device(scsidp); - up(&scst_mutex); TRACE_EXIT(); return; @@ -1137,6 +1197,16 @@ static int __init init_scst(void) (sizeof(cmd->sense_buffer) >= SCSI_SENSE_BUFFERSIZE)); } #endif + { + struct scst_tgt_dev *t; + BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn)); + } + + spin_lock_init(&scst_main_cmd_lists.cmd_list_lock); + INIT_LIST_HEAD(&scst_main_cmd_lists.active_cmd_list); + init_waitqueue_head(&scst_main_cmd_lists.cmd_list_waitQ); + list_add_tail(&scst_main_cmd_lists.lists_list_entry, + &scst_cmd_lists_list); scst_num_cpus = num_online_cpus(); @@ -1204,8 +1274,12 @@ static int __init init_scst(void) scst_scsi_op_list_init(); - for (i = 0; i < sizeof(scst_tasklets)/sizeof(scst_tasklets[0]); i++) - tasklet_init(&scst_tasklets[i], (void *)scst_cmd_tasklet, 0); + for (i = 0; i < sizeof(scst_tasklets)/sizeof(scst_tasklets[0]); i++) { + spin_lock_init(&scst_tasklets[i].tasklet_lock); + INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list); + tasklet_init(&scst_tasklets[i].tasklet, (void*)scst_cmd_tasklet, + (unsigned long)&scst_tasklets[i]); + } TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus, scst_threads); diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index 2ce4bcd57..9afc0df2b 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -84,21 +84,22 @@ void scst_set_cmd_error_sense(struct scst_cmd *cmd, uint8_t *sense, void scst_set_busy(struct scst_cmd *cmd) { + int c = cmd->sess->sess_cmd_count; + TRACE_ENTRY(); - if ((cmd->sess->sess_cmd_count <= 1) || - (cmd->sess->init_phase != SCST_SESS_IPH_READY)) + if ((c <= 1) || (cmd->sess->init_phase != SCST_SESS_IPH_READY)) { scst_set_cmd_error_status(cmd, SAM_STAT_BUSY); TRACE_MGMT_DBG("Sending BUSY status to initiator %s " "(cmds count %d, queue_type %x, sess->init_phase %d)", - cmd->sess->initiator_name, cmd->sess->sess_cmd_count, + cmd->sess->initiator_name, c, cmd->queue_type, cmd->sess->init_phase); } else { scst_set_cmd_error_status(cmd, SAM_STAT_TASK_SET_FULL); TRACE_MGMT_DBG("Sending QUEUE_FULL status to initiator %s " "(cmds count %d, queue_type %x, sess->init_phase %d)", - cmd->sess->initiator_name, cmd->sess->sess_cmd_count, + cmd->sess->initiator_name, c, cmd->queue_type, cmd->sess->init_phase); } @@ -218,7 +219,7 @@ out: return res; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ void scst_free_acg_dev(struct scst_acg_dev *acg_dev) { TRACE_ENTRY(); @@ -234,7 +235,7 @@ void scst_free_acg_dev(struct scst_acg_dev *acg_dev) return; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ struct scst_acg *scst_alloc_add_acg(const char *acg_name) { struct scst_acg *acg; @@ -260,7 +261,7 @@ out: return acg; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ int scst_destroy_acg(struct scst_acg *acg) { struct scst_acn *n, *nn; @@ -275,8 +276,6 @@ int scst_destroy_acg(struct scst_acg *acg) goto out; } - __scst_suspend_activity(); - TRACE_DBG("Removing acg %s from scst_acg_list", acg->acg_name); list_del(&acg->scst_acg_list_entry); @@ -295,8 +294,6 @@ int scst_destroy_acg(struct scst_acg *acg) scst_free_acg_dev(acg_dev); } - __scst_resume_activity(); - /* Freeing names */ list_for_each_entry_safe(n, nn, &acg->acn_list, acn_list_entry) @@ -342,8 +339,12 @@ static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess, tgt_dev->acg_dev = acg_dev; tgt_dev->sess = sess; - tgt_dev->cmd_count = 0; + atomic_set(&tgt_dev->cmd_count, 0); + atomic_set(&tgt_dev->curr_sn, 0); + tgt_dev->expected_sn = 1; + tgt_dev->p_cmd_lists = &scst_main_cmd_lists; + if (dev->scsi_dev != NULL) { TRACE(TRACE_DEBUG, "host=%d, channel=%d, id=%d, lun=%d, " "SCST lun=%Ld", dev->scsi_dev->host->host_no, @@ -361,6 +362,30 @@ static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess, INIT_LIST_HEAD(&tgt_dev->deferred_cmd_list); INIT_LIST_HEAD(&tgt_dev->skipped_sn_list); + if (dev->handler->parse_atomic && + sess->tgt->tgtt->preprocessing_done_atomic) { + if (sess->tgt->tgtt->rdy_to_xfer_atomic) + __set_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC, + &tgt_dev->tgt_dev_flags); + if (dev->handler->exec_atomic) + __set_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC, + &tgt_dev->tgt_dev_flags); + } + if (dev->handler->exec_atomic) { + if (sess->tgt->tgtt->rdy_to_xfer_atomic) + __set_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC, + &tgt_dev->tgt_dev_flags); + __set_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC, + &tgt_dev->tgt_dev_flags); + __set_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC, + &tgt_dev->tgt_dev_flags); + } + if (dev->handler->dev_done_atomic && + sess->tgt->tgtt->xmit_response_atomic) { + __set_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC, + &tgt_dev->tgt_dev_flags); + } + spin_lock_bh(&scst_temp_UA_lock); scst_set_sense(scst_temp_UA, sizeof(scst_temp_UA), SCST_LOAD_SENSE(scst_sense_reset_UA)); @@ -467,7 +492,7 @@ static void scst_free_tgt_dev(struct scst_tgt_dev *tgt_dev) return; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ int scst_sess_alloc_tgt_devs(struct scst_session *sess) { int res = 0; @@ -476,8 +501,6 @@ int scst_sess_alloc_tgt_devs(struct scst_session *sess) TRACE_ENTRY(); - __scst_suspend_activity(); - INIT_LIST_HEAD(&sess->sess_tgt_dev_list); list_for_each_entry(acg_dev, &sess->acg->acg_dev_list, acg_dev_list_entry) @@ -489,15 +512,13 @@ int scst_sess_alloc_tgt_devs(struct scst_session *sess) } } -out_resume: - __scst_resume_activity(); - +out: TRACE_EXIT(); return res; out_free: scst_sess_free_tgt_devs(sess); - goto out_resume; + goto out; } /* scst_mutex supposed to be held and activity suspended */ @@ -519,7 +540,7 @@ void scst_sess_free_tgt_devs(struct scst_session *sess) return; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ int scst_acg_add_dev(struct scst_acg *acg, struct scst_device *dev, lun_t lun, int read_only) { @@ -551,8 +572,6 @@ int scst_acg_add_dev(struct scst_acg *acg, struct scst_device *dev, lun_t lun, } acg_dev->rd_only_flag = read_only; - __scst_suspend_activity(); - TRACE_DBG("Adding acg_dev %p to acg_dev_list and dev_acg_dev_list", acg_dev); list_add_tail(&acg_dev->acg_dev_list_entry, &acg->acg_dev_list); @@ -569,9 +588,6 @@ int scst_acg_add_dev(struct scst_acg *acg, struct scst_device *dev, lun_t lun, &tmp_tgt_dev_list); } -out_resume: - __scst_resume_activity(); - out: TRACE_EXIT_RES(res); return res; @@ -583,10 +599,10 @@ out_free: scst_free_tgt_dev(tgt_dev); } scst_free_acg_dev(acg_dev); - goto out_resume; + goto out; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ int scst_acg_remove_dev(struct scst_acg *acg, struct scst_device *dev) { int res = 0; @@ -608,8 +624,6 @@ int scst_acg_remove_dev(struct scst_acg *acg, struct scst_device *dev) goto out; } - __scst_suspend_activity(); - list_for_each_entry_safe(tgt_dev, tt, &dev->dev_tgt_dev_list, dev_tgt_dev_list_entry) { @@ -618,8 +632,6 @@ int scst_acg_remove_dev(struct scst_acg *acg, struct scst_device *dev) } scst_free_acg_dev(acg_dev); - __scst_resume_activity(); - out: TRACE_EXIT_RES(res); return res; @@ -715,6 +727,7 @@ struct scst_cmd *scst_create_prepare_internal_cmd( goto out; } + res->cmd_lists = orig_cmd->cmd_lists; res->sess = orig_cmd->sess; res->state = SCST_CMD_STATE_SEND_TO_MIDLEV; res->atomic = scst_cmd_atomic(orig_cmd); @@ -750,9 +763,7 @@ void scst_free_internal_cmd(struct scst_cmd *cmd) { TRACE_ENTRY(); - if (cmd->bufflen > 0) - scst_release_space(cmd); - scst_destroy_cmd(cmd); + scst_cmd_put(cmd); TRACE_EXIT(); return; @@ -760,7 +771,7 @@ void scst_free_internal_cmd(struct scst_cmd *cmd) int scst_prepare_request_sense(struct scst_cmd *orig_cmd) { - int res = SCST_CMD_STATE_RES_RESTART; + int res = SCST_CMD_STATE_RES_CONT_NEXT; #define sbuf_size 252 static const unsigned char request_sense[6] = { REQUEST_SENSE, 0, 0, 0, sbuf_size, 0 }; @@ -776,9 +787,11 @@ int scst_prepare_request_sense(struct scst_cmd *orig_cmd) rs_cmd->cdb_len = sizeof(request_sense); rs_cmd->data_direction = SCST_DATA_READ; - spin_lock_irq(&scst_list_lock); - list_add(&rs_cmd->cmd_list_entry, &scst_active_cmd_list); - spin_unlock_irq(&scst_list_lock); + TRACE(TRACE_RETRY, "Adding REQUEST SENSE cmd %p to head of active " + "cmd list ", rs_cmd); + spin_lock_irq(&rs_cmd->cmd_lists->cmd_list_lock); + list_add(&rs_cmd->cmd_list_entry, &rs_cmd->cmd_lists->active_cmd_list); + spin_unlock_irq(&rs_cmd->cmd_lists->cmd_list_lock); out: TRACE_EXIT_RES(res); @@ -949,11 +962,12 @@ struct scst_session *scst_alloc_session(struct scst_tgt *tgt, int gfp_mask, sess->init_phase = SCST_SESS_IPH_INITING; atomic_set(&sess->refcnt, 0); INIT_LIST_HEAD(&sess->sess_tgt_dev_list); + spin_lock_init(&sess->sess_list_lock); INIT_LIST_HEAD(&sess->search_cmd_list); sess->tgt = tgt; INIT_LIST_HEAD(&sess->init_deferred_cmd_list); INIT_LIST_HEAD(&sess->init_deferred_mcmd_list); - + len = strlen(initiator_name); nm = kmalloc(len + 1, gfp_mask); if (nm == NULL) { @@ -978,19 +992,20 @@ void scst_free_session(struct scst_session *sess) { TRACE_ENTRY(); + scst_suspend_activity(); down(&scst_mutex); + TRACE_DBG("Removing sess %p from the list", sess); list_del(&sess->sess_list_entry); TRACE_DBG("Removing session %p from acg %s", sess, sess->acg->acg_name); list_del(&sess->acg_sess_list_entry); - - __scst_suspend_activity(); + scst_sess_free_tgt_devs(sess); - __scst_resume_activity(); wake_up_all(&sess->tgt->unreg_waitQ); up(&scst_mutex); + scst_resume_activity(); kfree(sess->initiator_name); kmem_cache_free(scst_sess_cachep, sess); @@ -1059,6 +1074,8 @@ struct scst_cmd *scst_alloc_cmd(int gfp_mask) memset(cmd, 0, sizeof(*cmd)); #endif + atomic_set(&cmd->cmd_ref, 1); + cmd->cmd_lists = &scst_main_cmd_lists; cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED; cmd->timeout = SCST_DEFAULT_TIMEOUT; cmd->retries = 1; @@ -1071,19 +1088,19 @@ out: return cmd; } -static void scst_destroy_put_cmd(struct scst_cmd *cmd) +void scst_destroy_put_cmd(struct scst_cmd *cmd) { scst_sess_put(cmd->sess); /* At this point tgt_dev can be dead, but the pointer remains not-NULL */ if (likely(cmd->tgt_dev != NULL)) - scst_dec_cmd_count(); + __scst_put(); scst_destroy_cmd(cmd); return; } -/* No locks supposed to be held. Must be called only from scst_finish_cmd()! */ +/* No locks supposed to be held */ void scst_free_cmd(struct scst_cmd *cmd) { int destroy = 1; @@ -1100,6 +1117,24 @@ void scst_free_cmd(struct scst_cmd *cmd) } #endif + if (likely(cmd->tgt_dev != NULL)) + atomic_dec(&cmd->tgt_dev->cmd_count); + + /* + * cmd->mgmt_cmnd can't being changed here, since for that it either + * must be on search_cmd_list, or cmd_ref must be taken. Both are + * false here. + */ + if (unlikely(cmd->mgmt_cmnd)) + scst_complete_cmd_mgmt(cmd, cmd->mgmt_cmnd); + + if (unlikely(cmd->internal)) { + if (cmd->bufflen > 0) + scst_release_space(cmd); + scst_destroy_cmd(cmd); + goto out; + } + if (cmd->tgtt->on_free_cmd != NULL) { TRACE_DBG("Calling target's on_free_cmd(%p)", cmd); cmd->tgtt->on_free_cmd(cmd); @@ -1126,34 +1161,29 @@ void scst_free_cmd(struct scst_cmd *cmd) "target %s, lun %Ld, sn %d, expected_sn %d)", cmd->cdb[0], cmd->tgtt->name, (uint64_t)cmd->lun, cmd->sn, cmd->tgt_dev->expected_sn); - scst_inc_expected_sn_unblock(cmd->tgt_dev, cmd, 0); + scst_inc_expected_sn_unblock(cmd->tgt_dev, cmd); } #endif - if (unlikely(test_bit(SCST_CMD_OUT_OF_SN, - &cmd->cmd_flags))) - { - spin_lock_bh(&cmd->tgt_dev->sn_lock); - set_bit(SCST_CMD_CAN_BE_DESTROYED, - &cmd->cmd_flags); - barrier(); /* to reread SCST_CMD_OUT_OF_SN */ - destroy = !test_bit(SCST_CMD_OUT_OF_SN, - &cmd->cmd_flags); + + if (unlikely(cmd->out_of_sn)) { TRACE(TRACE_SCSI_SERIALIZING, "Out of SN " "cmd %p (tag %d, sn %d), destroy=%d", cmd, cmd->tag, cmd->sn, destroy); - spin_unlock_bh(&cmd->tgt_dev->sn_lock); + destroy = test_and_set_bit(SCST_CMD_CAN_BE_DESTROYED, + &cmd->cmd_flags); } } if (likely(destroy)) scst_destroy_put_cmd(cmd); +out: TRACE_EXIT(); return; } /* No locks supposed to be held. */ -void scst_check_retries(struct scst_tgt *tgt, int processible_env) +void scst_check_retries(struct scst_tgt *tgt) { int need_wake_up = 0; @@ -1174,29 +1204,25 @@ void scst_check_retries(struct scst_tgt *tgt, int processible_env) tgt->retry_cmds); spin_lock_irqsave(&tgt->tgt_lock, flags); - spin_lock(&scst_list_lock); - list_for_each_entry_safe(c, tc, &tgt->retry_cmd_list, cmd_list_entry) { tgt->retry_cmds--; - TRACE(TRACE_RETRY, "Moving retry cmd %p to active cmd " - "list (retry_cmds left %d)", c, tgt->retry_cmds); - list_move(&c->cmd_list_entry, &scst_active_cmd_list); + TRACE(TRACE_RETRY, "Moving retry cmd %p to head of active " + "cmd list (retry_cmds left %d)", c, tgt->retry_cmds); + spin_lock(&c->cmd_lists->cmd_list_lock); + list_move(&c->cmd_list_entry, &c->cmd_lists->active_cmd_list); + wake_up(&c->cmd_lists->cmd_list_waitQ); + spin_unlock(&c->cmd_lists->cmd_list_lock); need_wake_up++; if (need_wake_up >= 2) /* "slow start" */ break; } - - spin_unlock(&scst_list_lock); spin_unlock_irqrestore(&tgt->tgt_lock, flags); } - if (need_wake_up && !processible_env) - wake_up(&scst_list_waitQ); - TRACE_EXIT(); return; } @@ -1213,7 +1239,7 @@ void scst_tgt_retry_timer_fn(unsigned long arg) tgt->retry_timer_active = 0; spin_unlock_irqrestore(&tgt->tgt_lock, flags); - scst_check_retries(tgt, 0); + scst_check_retries(tgt); TRACE_EXIT(); return; @@ -1238,22 +1264,20 @@ out: return mcmd; } -void scst_free_mgmt_cmd(struct scst_mgmt_cmd *mcmd, int del) +void scst_free_mgmt_cmd(struct scst_mgmt_cmd *mcmd) { unsigned long flags; TRACE_ENTRY(); - spin_lock_irqsave(&scst_list_lock, flags); - if (del) - list_del(&mcmd->mgmt_cmd_list_entry); + spin_lock_irqsave(&mcmd->sess->sess_list_lock, flags); mcmd->sess->sess_cmd_count--; - spin_unlock_irqrestore(&scst_list_lock, flags); + spin_unlock_irqrestore(&mcmd->sess->sess_list_lock, flags); scst_sess_put(mcmd->sess); if (mcmd->mcmd_tgt_dev != NULL) - scst_dec_cmd_count(); + __scst_put(); mempool_free(mcmd, scst_mgmt_mempool); @@ -1722,7 +1746,6 @@ void scst_process_reset(struct scst_device *dev, { struct scst_tgt_dev *tgt_dev; struct scst_cmd *cmd, *tcmd; - int wake = 0; TRACE_ENTRY(); @@ -1759,8 +1782,7 @@ void scst_process_reset(struct scst_device *dev, { struct scst_session *sess = tgt_dev->sess; - local_bh_disable(); - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&sess->sess_list_lock); TRACE_DBG("Searching in search cmd list (sess=%p)", sess); list_for_each_entry(cmd, &sess->search_cmd_list, @@ -1774,27 +1796,23 @@ void scst_process_reset(struct scst_device *dev, (tgt_dev->sess != originator), 0); } } - spin_unlock_irq(&scst_list_lock); - local_bh_enable(); + spin_unlock_irq(&sess->sess_list_lock); } list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list, blocked_cmd_list_entry) { if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) { list_del(&cmd->blocked_cmd_list_entry); - TRACE_MGMT_DBG("Moving aborted blocked cmd %p " + TRACE_MGMT_DBG("Adding aborted blocked cmd %p " "to active cmd list", cmd); - spin_lock_irq(&scst_list_lock); - list_move_tail(&cmd->cmd_list_entry, - &scst_active_cmd_list); - spin_unlock_irq(&scst_list_lock); - wake = 1; + spin_lock_irq(&cmd->cmd_lists->cmd_list_lock); + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock); } } - if (wake) - wake_up(&scst_list_waitQ); - TRACE_EXIT(); return; } @@ -1992,12 +2010,9 @@ restart: tcmd->tag, tcmd->sn); tgt_dev->def_cmd_count--; list_del(&tcmd->sn_cmd_list_entry); - if (test_bit(SCST_CMD_CAN_BE_DESTROYED, + if (test_and_set_bit(SCST_CMD_CAN_BE_DESTROYED, &tcmd->cmd_flags)) { scst_destroy_put_cmd(tcmd); - } else { - smp_mb__before_clear_bit(); - clear_bit(SCST_CMD_OUT_OF_SN, &tcmd->cmd_flags); } expected_sn = __scst_inc_expected_sn(tgt_dev); goto restart; @@ -2091,10 +2106,10 @@ void scst_unblock_cmds(struct scst_device *dev) { #ifdef STRICT_SERIALIZING struct scst_cmd *cmd, *t; - int found = 0; TRACE_ENTRY(); + local_irq_save(flags); list_for_each_entry_safe(cmd, t, &dev->blocked_cmd_list, blocked_cmd_list_entry) { unsigned long flags; @@ -2117,62 +2132,64 @@ void scst_unblock_cmds(struct scst_device *dev) } list_del(&cmd->blocked_cmd_list_entry); - TRACE_MGMT_DBG("Moving cmd %p to active cmd list", cmd); - spin_lock_irqsave(&scst_list_lock, flags); - list_move(&cmd->cmd_list_entry, &scst_active_cmd_list); - spin_unlock_irqrestore(&scst_list_lock, flags); - found = 1; + TRACE_MGMT_DBG("Adding cmd %p to head of active cmd list", cmd); + spin_lock(&cmd->cmd_lists->cmd_list_lock); + list_add(&cmd->cmd_list_entry, &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock(&cmd->cmd_lists->cmd_list_lock); if (brk) break; } - if (found) - wake_up(&scst_list_waitQ); + local_irq_restore(, flags); #else /* STRICT_SERIALIZING */ struct scst_cmd *cmd, *tcmd; unsigned long flags; TRACE_ENTRY(); - spin_lock_irqsave(&scst_list_lock, flags); + local_irq_save(flags); list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list, blocked_cmd_list_entry) { list_del(&cmd->blocked_cmd_list_entry); - TRACE_MGMT_DBG("Moving blocked cmd %p to active cmd list", cmd); - list_move_tail(&cmd->cmd_list_entry, &scst_active_cmd_list); - wake_up(&scst_list_waitQ); + TRACE_MGMT_DBG("Adding blocked cmd %p to active cmd list", cmd); + spin_lock(&cmd->cmd_lists->cmd_list_lock); + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock(&cmd->cmd_lists->cmd_list_lock); } - spin_unlock_irqrestore(&scst_list_lock, flags); + local_irq_restore(flags); #endif /* STRICT_SERIALIZING */ TRACE_EXIT(); return; } -static struct scst_cmd *scst_inc_expected_sn( +static struct scst_cmd *__scst_inc_expected_sn_unblock( struct scst_tgt_dev *tgt_dev, struct scst_cmd *out_of_sn_cmd) { struct scst_cmd *res = NULL; if (out_of_sn_cmd->sn == tgt_dev->expected_sn) { __scst_inc_expected_sn(tgt_dev); + res = scst_check_deferred_commands(tgt_dev, + tgt_dev->expected_sn); } else { + out_of_sn_cmd->out_of_sn = 1; spin_lock_bh(&tgt_dev->sn_lock); tgt_dev->def_cmd_count++; - set_bit(SCST_CMD_OUT_OF_SN, &out_of_sn_cmd->cmd_flags); list_add_tail(&out_of_sn_cmd->sn_cmd_list_entry, &tgt_dev->skipped_sn_list); TRACE(TRACE_SCSI_SERIALIZING, "out_of_sn_cmd %p with sn %d " "added to skipped_sn_list (expected_sn %d)", out_of_sn_cmd, out_of_sn_cmd->sn, tgt_dev->expected_sn); spin_unlock_bh(&tgt_dev->sn_lock); - smp_mb(); /* just in case, we need new value of tgt_dev->expected_sn */ } - res = scst_check_deferred_commands(tgt_dev, tgt_dev->expected_sn); return res; } void scst_inc_expected_sn_unblock(struct scst_tgt_dev *tgt_dev, - struct scst_cmd *out_of_sn_cmd, int locked) + struct scst_cmd *out_of_sn_cmd) { struct scst_cmd *cmd; @@ -2183,18 +2200,15 @@ void scst_inc_expected_sn_unblock(struct scst_tgt_dev *tgt_dev, goto out; } - cmd = scst_inc_expected_sn(tgt_dev, out_of_sn_cmd); + cmd = __scst_inc_expected_sn_unblock(tgt_dev, out_of_sn_cmd); if (cmd != NULL) { - unsigned long flags = 0; - if (!locked) - spin_lock_irqsave(&scst_list_lock, flags); + unsigned long flags; + spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags); TRACE(TRACE_SCSI_SERIALIZING, "cmd %p with sn %d " - "moved to active cmd list", cmd, cmd->sn); - list_move(&cmd->cmd_list_entry, &scst_active_cmd_list); - if (!locked) - spin_unlock_irqrestore(&scst_list_lock, flags); - if (!out_of_sn_cmd->processible_env) - wake_up(&scst_list_waitQ); + "added to the head of active cmd list", cmd, cmd->sn); + list_add(&cmd->cmd_list_entry, &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags); } out: @@ -2223,7 +2237,6 @@ void __init scst_scsi_op_list_init(void) return; } - #ifdef DEBUG /* Original taken from the XFS code */ unsigned long scst_random(void) @@ -2263,111 +2276,140 @@ unsigned long scst_random(void) static void tm_dbg_timer_fn(unsigned long arg); -/* All serialized by scst_list_lock */ -static int tm_dbg_release; -static int tm_dbg_blocked; +static spinlock_t scst_tm_dbg_lock = SPIN_LOCK_UNLOCKED; +/* All serialized by scst_tm_dbg_lock */ +struct +{ + unsigned int tm_dbg_active:1; + unsigned int tm_dbg_release:1; + unsigned int tm_dbg_blocked:1; +} tm_dbg_flags; static LIST_HEAD(tm_dbg_delayed_cmd_list); static int tm_dbg_delayed_cmds_count; static int tm_dbg_passed_cmds_count; static int tm_dbg_state; static int tm_dbg_on_state_passes; static DEFINE_TIMER(tm_dbg_timer, tm_dbg_timer_fn, 0, 0); +static wait_queue_head_t *tm_dbg_p_cmd_list_waitQ; static const int tm_dbg_on_state_num_passes[] = { 5, 1, 0x7ffffff }; void tm_dbg_init_tgt_dev(struct scst_tgt_dev *tgt_dev, struct scst_acg_dev *acg_dev) { - if ((acg_dev->acg == scst_default_acg) && (acg_dev->lun == 0)) { - /* Do TM debugging only for LUN 0 */ - tm_dbg_state = INIT_TM_DBG_STATE; - tm_dbg_on_state_passes = - tm_dbg_on_state_num_passes[tm_dbg_state]; - __set_bit(SCST_TGT_DEV_UNDER_TM_DBG, &tgt_dev->tgt_dev_flags); - PRINT_INFO("LUN 0 connected from initiator %s is under " - "TM debugging", tgt_dev->sess->tgt->tgtt->name); + if (((acg_dev->acg == scst_default_acg) || + (acg_dev->acg == tgt_dev->sess->tgt->default_acg)) && + (acg_dev->lun == 0)) { + unsigned long flags; + spin_lock_irqsave(&scst_tm_dbg_lock, flags); + if (!tm_dbg_flags.tm_dbg_active) { + /* Do TM debugging only for LUN 0 */ + tm_dbg_p_cmd_list_waitQ = + &tgt_dev->p_cmd_lists->cmd_list_waitQ; + tm_dbg_state = INIT_TM_DBG_STATE; + tm_dbg_on_state_passes = + tm_dbg_on_state_num_passes[tm_dbg_state]; + __set_bit(SCST_TGT_DEV_UNDER_TM_DBG, &tgt_dev->tgt_dev_flags); + PRINT_INFO("LUN 0 connected from initiator %s is under " + "TM debugging", tgt_dev->sess->tgt->tgtt->name); + tm_dbg_flags.tm_dbg_active = 1; + } + spin_unlock_irqrestore(&scst_tm_dbg_lock, flags); } } void tm_dbg_deinit_tgt_dev(struct scst_tgt_dev *tgt_dev) { - if (test_bit(SCST_TGT_DEV_UNDER_TM_DBG, &tgt_dev->tgt_dev_flags)) + if (test_bit(SCST_TGT_DEV_UNDER_TM_DBG, &tgt_dev->tgt_dev_flags)) { + unsigned long flags; del_timer_sync(&tm_dbg_timer); + spin_lock_irqsave(&scst_tm_dbg_lock, flags); + tm_dbg_p_cmd_list_waitQ = NULL; + tm_dbg_flags.tm_dbg_active = 0; + spin_unlock_irqrestore(&scst_tm_dbg_lock, flags); + } } static void tm_dbg_timer_fn(unsigned long arg) { - TRACE_MGMT_DBG("%s: delayed cmd timer expired", __func__); - tm_dbg_release = 1; + TRACE_MGMT_DBG("%s", "delayed cmd timer expired"); + tm_dbg_flags.tm_dbg_release = 1; smp_mb(); - wake_up_all(&scst_list_waitQ); + wake_up_all(tm_dbg_p_cmd_list_waitQ); } -/* Called under scst_list_lock */ +/* Called under scst_tm_dbg_lock and IRQs off */ static void tm_dbg_delay_cmd(struct scst_cmd *cmd) { switch(tm_dbg_state) { case TM_DBG_STATE_ABORT: if (tm_dbg_delayed_cmds_count == 0) { unsigned long d = 58*HZ + (scst_random() % (4*HZ)); - TRACE_MGMT_DBG("%s: delaying timed cmd %p (tag %d) " + TRACE_MGMT_DBG("STATE ABORT: delaying cmd %p (tag %d) " "for %ld.%ld seconds (%ld HZ), " - "tm_dbg_on_state_passes=%d", __func__, cmd, - cmd->tag, d/HZ, (d%HZ)*100/HZ, d, - tm_dbg_on_state_passes); + "tm_dbg_on_state_passes=%d", cmd, cmd->tag, + d/HZ, (d%HZ)*100/HZ, d, tm_dbg_on_state_passes); mod_timer(&tm_dbg_timer, jiffies + d); #if 0 - tm_dbg_blocked = 1; + tm_dbg_flags.tm_dbg_blocked = 1; #endif } else { - TRACE_MGMT_DBG("%s: delaying another timed cmd %p " + TRACE_MGMT_DBG("Delaying another timed cmd %p " "(tag %d), delayed_cmds_count=%d, " - "tm_dbg_on_state_passes=%d", __func__, cmd, - cmd->tag, tm_dbg_delayed_cmds_count, + "tm_dbg_on_state_passes=%d", cmd, cmd->tag, + tm_dbg_delayed_cmds_count, tm_dbg_on_state_passes); if (tm_dbg_delayed_cmds_count == 2) - tm_dbg_blocked = 0; + tm_dbg_flags.tm_dbg_blocked = 0; } break; case TM_DBG_STATE_RESET: case TM_DBG_STATE_OFFLINE: - TRACE_MGMT_DBG("%s: delaying cmd %p " + TRACE_MGMT_DBG("STATE RESET/OFFLINE: delaying cmd %p " "(tag %d), delayed_cmds_count=%d, " - "tm_dbg_on_state_passes=%d", __func__, cmd, - cmd->tag, tm_dbg_delayed_cmds_count, - tm_dbg_on_state_passes); - tm_dbg_blocked = 1; + "tm_dbg_on_state_passes=%d", cmd, cmd->tag, + tm_dbg_delayed_cmds_count, tm_dbg_on_state_passes); + tm_dbg_flags.tm_dbg_blocked = 1; break; default: sBUG(); } - list_move_tail(&cmd->cmd_list_entry, &tm_dbg_delayed_cmd_list); + /* IRQs already off */ + spin_lock(&cmd->cmd_lists->cmd_list_lock); + list_add_tail(&cmd->cmd_list_entry, &tm_dbg_delayed_cmd_list); + spin_unlock(&cmd->cmd_lists->cmd_list_lock); cmd->tm_dbg_delayed = 1; tm_dbg_delayed_cmds_count++; return; } -/* Called under scst_list_lock */ +/* No locks */ void tm_dbg_check_released_cmds(void) { - if (tm_dbg_release) { + if (tm_dbg_flags.tm_dbg_release) { struct scst_cmd *cmd, *tc; + spin_lock_irq(&scst_tm_dbg_lock); list_for_each_entry_safe_reverse(cmd, tc, &tm_dbg_delayed_cmd_list, cmd_list_entry) { - TRACE_MGMT_DBG("%s: Releasing timed cmd %p " - "(tag %d), delayed_cmds_count=%d", __func__, - cmd, cmd->tag, tm_dbg_delayed_cmds_count); - list_move(&cmd->cmd_list_entry, &scst_active_cmd_list); + TRACE_MGMT_DBG("Releasing timed cmd %p (tag %d), " + "delayed_cmds_count=%d", cmd, cmd->tag, + tm_dbg_delayed_cmds_count); + spin_lock(&cmd->cmd_lists->cmd_list_lock); + list_move(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + spin_unlock(&cmd->cmd_lists->cmd_list_lock); } - tm_dbg_release = 0; + tm_dbg_flags.tm_dbg_release = 0; + spin_unlock_irq(&scst_tm_dbg_lock); } } +/* Called under scst_tm_dbg_lock */ static void tm_dbg_change_state(void) { - tm_dbg_blocked = 0; + tm_dbg_flags.tm_dbg_blocked = 0; if (--tm_dbg_on_state_passes == 0) { switch(tm_dbg_state) { case TM_DBG_STATE_ABORT: @@ -2375,7 +2417,7 @@ static void tm_dbg_change_state(void) "tm_dbg_state to RESET"); tm_dbg_state = TM_DBG_STATE_RESET; - tm_dbg_blocked = 0; + tm_dbg_flags.tm_dbg_blocked = 0; break; case TM_DBG_STATE_RESET: case TM_DBG_STATE_OFFLINE: @@ -2402,15 +2444,17 @@ static void tm_dbg_change_state(void) del_timer(&tm_dbg_timer); } -/* Called under scst_list_lock */ +/* No locks */ int tm_dbg_check_cmd(struct scst_cmd *cmd) { int res = 0; + unsigned long flags; if (cmd->tm_dbg_immut) goto out; if (cmd->tm_dbg_delayed) { + spin_lock_irqsave(&scst_tm_dbg_lock, flags); TRACE_MGMT_DBG("Processing delayed cmd %p (tag %d), " "delayed_cmds_count=%d", cmd, cmd->tag, tm_dbg_delayed_cmds_count); @@ -2420,25 +2464,31 @@ int tm_dbg_check_cmd(struct scst_cmd *cmd) if ((tm_dbg_delayed_cmds_count == 0) && (tm_dbg_state == TM_DBG_STATE_ABORT)) tm_dbg_change_state(); - + spin_unlock_irqrestore(&scst_tm_dbg_lock, flags); } else if (cmd->tgt_dev && test_bit(SCST_TGT_DEV_UNDER_TM_DBG, &cmd->tgt_dev->tgt_dev_flags)) { /* Delay 50th command */ - if (tm_dbg_blocked || (++tm_dbg_passed_cmds_count % 50) == 0) { + spin_lock_irqsave(&scst_tm_dbg_lock, flags); + if (tm_dbg_flags.tm_dbg_blocked || + (++tm_dbg_passed_cmds_count % 50) == 0) { tm_dbg_delay_cmd(cmd); res = 1; } else cmd->tm_dbg_immut = 1; + spin_unlock_irqrestore(&scst_tm_dbg_lock, flags); } out: return res; } -/* Called under scst_list_lock */ +/* No locks */ void tm_dbg_release_cmd(struct scst_cmd *cmd) { struct scst_cmd *c; + unsigned long flags; + + spin_lock_irqsave(&scst_tm_dbg_lock, flags); list_for_each_entry(c, &tm_dbg_delayed_cmd_list, cmd_list_entry) { if (c == cmd) { @@ -2446,30 +2496,38 @@ void tm_dbg_release_cmd(struct scst_cmd *cmd) "delayed cmd %p (tag=%d), moving it to " "active cmd list (delayed_cmds_count=%d)", c, c->tag, tm_dbg_delayed_cmds_count); - list_move(&c->cmd_list_entry, &scst_active_cmd_list); - wake_up_all(&scst_list_waitQ); + spin_lock(&cmd->cmd_lists->cmd_list_lock); + list_move(&c->cmd_list_entry, + &c->cmd_lists->active_cmd_list); + wake_up(&c->cmd_lists->cmd_list_waitQ); + spin_unlock(&cmd->cmd_lists->cmd_list_lock); break; } } + spin_unlock_irqrestore(&scst_tm_dbg_lock, flags); } -/* Called under scst_list_lock */ -void tm_dbg_task_mgmt(const char *fn) +/* No locks */ +void tm_dbg_task_mgmt(const char *fn, int force) { - if (tm_dbg_state != TM_DBG_STATE_OFFLINE) { + unsigned long flags; + + spin_lock_irqsave(&scst_tm_dbg_lock, flags); + if ((tm_dbg_state != TM_DBG_STATE_OFFLINE) || force) { TRACE_MGMT_DBG("%s: freeing %d delayed cmds", fn, tm_dbg_delayed_cmds_count); tm_dbg_change_state(); - tm_dbg_release = 1; + tm_dbg_flags.tm_dbg_release = 1; smp_mb(); - wake_up_all(&scst_list_waitQ); + wake_up_all(tm_dbg_p_cmd_list_waitQ); } else { TRACE_MGMT_DBG("%s: while OFFLINE state, doing nothing", fn); } + spin_unlock_irqrestore(&scst_tm_dbg_lock, flags); } int tm_dbg_is_release(void) { - return tm_dbg_release; + return tm_dbg_flags.tm_dbg_release; } #endif /* DEBUG_TM */ diff --git a/scst/src/scst_priv.h b/scst/src/scst_priv.h index bf242893a..078a1ff3d 100644 --- a/scst/src/scst_priv.h +++ b/scst/src/scst_priv.h @@ -70,8 +70,15 @@ extern unsigned long scst_trace_flag; ** Bits for scst_flags **/ -/* Set if new commands initialization should be suspended for a while */ -#define SCST_FLAG_SUSPENDED 0 +/* + * Set if new commands initialization is being suspended for a while. + * Used to let TM commands execute while preparing the suspend, since + * RESET or ABORT could be necessary to free SCSI commands. + */ +#define SCST_FLAG_SUSPENDING 0 + +/* Set if new commands initialization is suspended for a while */ +#define SCST_FLAG_SUSPENDED 1 /* Set if a TM command is being performed */ #define SCST_FLAG_TM_ACTIVE 2 @@ -129,13 +136,13 @@ extern struct kmem_cache *scst_tgtd_cachep; #define SCST_ACG_DEV_CACHE_STRING "scst_acg_dev" extern struct kmem_cache *scst_acgd_cachep; +extern spinlock_t scst_main_lock; + extern struct scst_sgv_pools scst_sgv; extern unsigned long scst_flags; extern struct semaphore scst_mutex; extern atomic_t scst_cmd_count; -extern spinlock_t scst_list_lock; -extern struct list_head scst_dev_wait_sess_list; /* protected by scst_list_lock */ extern struct list_head scst_template_list; /* protected by scst_mutex */ extern struct list_head scst_dev_list; /* protected by scst_mutex */ extern struct list_head scst_dev_type_list; /* protected by scst_mutex */ @@ -144,10 +151,12 @@ extern wait_queue_head_t scst_dev_cmd_waitQ; extern struct list_head scst_acg_list; extern struct scst_acg *scst_default_acg; -/* The following lists protected by scst_list_lock */ -extern struct list_head scst_active_cmd_list; +extern spinlock_t scst_init_lock; extern struct list_head scst_init_cmd_list; -extern struct list_head scst_cmd_list; +extern wait_queue_head_t scst_init_cmd_list_waitQ; +extern unsigned int scst_init_poll_cnt; + +extern struct scst_cmd_lists scst_main_cmd_lists; extern spinlock_t scst_cmd_mem_lock; extern unsigned long scst_max_cmd_mem, scst_cur_max_cmd_mem, scst_cur_cmd_mem; @@ -157,16 +166,21 @@ extern struct work_struct scst_cmd_mem_work; extern struct delayed_work scst_cmd_mem_work; #endif -/* The following lists protected by scst_list_lock as well */ -extern struct list_head scst_mgmt_cmd_list; +extern spinlock_t scst_mcmd_lock; +/* The following lists protected by scst_mcmd_lock */ extern struct list_head scst_active_mgmt_cmd_list; extern struct list_head scst_delayed_mgmt_cmd_list; -extern struct tasklet_struct scst_tasklets[NR_CPUS]; -extern wait_queue_head_t scst_list_waitQ; - extern wait_queue_head_t scst_mgmt_cmd_list_waitQ; +struct scst_tasklet +{ + spinlock_t tasklet_lock; + struct list_head tasklet_cmd_list; + struct tasklet_struct tasklet; +}; +extern struct scst_tasklet scst_tasklets[NR_CPUS]; + extern wait_queue_head_t scst_mgmt_waitQ; extern spinlock_t scst_mgmt_lock; extern struct list_head scst_sess_mgmt_list; @@ -180,6 +194,7 @@ struct scst_threads_info_t { struct semaphore cmd_threads_mutex; u32 nr_cmd_threads; struct list_head cmd_threads_list; + struct task_struct *init_cmd_thread; struct task_struct *mgmt_thread; struct task_struct *mgmt_cmd_thread; }; @@ -223,10 +238,11 @@ static inline int __scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev) } void scst_inc_expected_sn_unblock(struct scst_tgt_dev *tgt_dev, - struct scst_cmd *cmd_sn, int locked); + struct scst_cmd *cmd_sn); int scst_cmd_thread(void *arg); void scst_cmd_tasklet(long p); +int scst_init_cmd_thread(void *arg); int scst_mgmt_cmd_thread(void *arg); int scst_mgmt_thread(void *arg); #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) @@ -276,7 +292,7 @@ static inline void scst_destroy_cmd(struct scst_cmd *cmd) void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context, int check_retries); -void scst_check_retries(struct scst_tgt *tgt, int processible_env); +void scst_check_retries(struct scst_tgt *tgt); void scst_tgt_retry_timer_fn(unsigned long arg); #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) @@ -324,7 +340,8 @@ struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess, uint32_t tag); struct scst_mgmt_cmd *scst_alloc_mgmt_cmd(int gfp_mask); -void scst_free_mgmt_cmd(struct scst_mgmt_cmd *mcmd, int del); +void scst_free_mgmt_cmd(struct scst_mgmt_cmd *mcmd); +void scst_complete_cmd_mgmt(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd); /* /proc support */ int scst_proc_init_module(void); @@ -438,21 +455,24 @@ static inline void scst_dec_on_dev_cmd(struct scst_cmd *cmd) wake_up_all(&cmd->dev->on_dev_waitQ); } -static inline void scst_inc_cmd_count(void) +static inline void __scst_get(int barrier) { atomic_inc(&scst_cmd_count); - /* It's needed to be before test_bit(SCST_FLAG_SUSPENDED) */ - smp_mb__after_atomic_inc(); TRACE_DBG("Incrementing scst_cmd_count(%d)", - atomic_read(&scst_cmd_count)); + atomic_read(&scst_cmd_count)); + + if (barrier) + smp_mb__after_atomic_inc(); } -static inline void scst_dec_cmd_count(void) +static inline void __scst_put(void) { int f; f = atomic_dec_and_test(&scst_cmd_count); - if (f && unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) + if (f && unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) { + TRACE_MGMT_DBG("%s", "Waking up scst_dev_cmd_waitQ"); wake_up_all(&scst_dev_cmd_waitQ); + } TRACE_DBG("Decrementing scst_cmd_count(%d)", atomic_read(&scst_cmd_count)); } @@ -470,8 +490,16 @@ static inline void scst_sess_put(struct scst_session *sess) scst_sched_session_free(sess); } -void __scst_suspend_activity(void); -void __scst_resume_activity(void); +static inline void scst_cmd_get(struct scst_cmd *cmd) +{ + atomic_inc(&cmd->cmd_ref); +} + +static inline void scst_cmd_put(struct scst_cmd *cmd) +{ + if (atomic_dec_and_test(&cmd->cmd_ref)) + scst_free_cmd(cmd); +} extern void scst_throttle_cmd(struct scst_cmd *cmd); extern void scst_unthrottle_cmd(struct scst_cmd *cmd); @@ -504,7 +532,7 @@ extern void tm_dbg_deinit_tgt_dev(struct scst_tgt_dev *tgt_dev); extern void tm_dbg_check_released_cmds(void); extern int tm_dbg_check_cmd(struct scst_cmd *cmd); extern void tm_dbg_release_cmd(struct scst_cmd *cmd); -extern void tm_dbg_task_mgmt(const char *fn); +extern void tm_dbg_task_mgmt(const char *fn, int force); extern int tm_dbg_is_release(void); #else static inline void tm_dbg_init_tgt_dev(struct scst_tgt_dev *tgt_dev, @@ -516,7 +544,7 @@ static inline int tm_dbg_check_cmd(struct scst_cmd *cmd) return 0; } static inline void tm_dbg_release_cmd(struct scst_cmd *cmd) {} -static inline void tm_dbg_task_mgmt(const char *fn) {} +static inline void tm_dbg_task_mgmt(const char *fn, int force) {} static inline int tm_dbg_is_release(void) { return 0; diff --git a/scst/src/scst_proc.c b/scst/src/scst_proc.c index 7d2d1a878..f5038b460 100644 --- a/scst/src/scst_proc.c +++ b/scst/src/scst_proc.c @@ -474,7 +474,7 @@ static void scst_proc_del_acg_tree(struct proc_dir_entry *acg_proc_root, return; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ static int scst_proc_group_add(const char *p) { int res = 0, len = strlen(p) + 1; @@ -515,7 +515,7 @@ out_nomem: goto out; } -/* scst_mutex supposed to be held */ +/* The activity supposed to be suspended and scst_mutex held */ static int scst_proc_del_free_acg(struct scst_acg *acg, int remove_proc) { const char *name; @@ -1136,9 +1136,11 @@ static int scst_proc_scsi_tgt_gen_write(struct file *file, const char __user *bu goto out_free; } + scst_suspend_activity(); + if (down_interruptible(&scst_mutex) != 0) { res = -EINTR; - goto out_free; + goto out_free_resume; } switch (action) { @@ -1188,14 +1190,18 @@ static int scst_proc_scsi_tgt_gen_write(struct file *file, const char __user *bu out_up_free: up(&scst_mutex); +out_free_resume: + scst_resume_activity(); + out_free: free_page((unsigned long)buffer); + out: TRACE_EXIT_RES(res); return res; } -/* Called under scst_mutex */ +/* The activity supposed to be suspended and scst_mutex held */ static int scst_proc_assign_handler(char *buf) { int res = 0; @@ -1354,9 +1360,11 @@ static int scst_proc_groups_devices_write(struct file *file, const char __user * goto out_free; } + scst_suspend_activity(); + if (down_interruptible(&scst_mutex) != 0) { res = -EINTR; - goto out_free; + goto out_free_resume; } switch (action) { @@ -1476,8 +1484,12 @@ static int scst_proc_groups_devices_write(struct file *file, const char __user * out_free_up: up(&scst_mutex); +out_free_resume: + scst_resume_activity(); + out_free: free_page((unsigned long)buffer); + out: TRACE_EXIT_RES(res); return res; @@ -1674,12 +1686,12 @@ static int scst_sessions_info_show(struct seq_file *seq, void *v) goto out; } - seq_printf(seq, "%-20s%-35s%-20s%-15s\n", "Target name", "Initiator name", + seq_printf(seq, "%-20s%-35s%-35s%-15s\n", "Target name", "Initiator name", "Group name", "Command Count"); list_for_each_entry(acg, &scst_acg_list, scst_acg_list_entry) { list_for_each_entry(sess, &acg->acg_sess_list, acg_sess_list_entry) { - seq_printf(seq, "%-20s%-35s%-20s%-15d\n", + seq_printf(seq, "%-20s%-35s%-35s%-15d\n", sess->tgt->tgtt->name, sess->initiator_name, acg->acg_name, diff --git a/scst/src/scst_targ.c b/scst/src/scst_targ.c index 9c66c23ad..661d9a311 100644 --- a/scst/src/scst_targ.c +++ b/scst/src/scst_targ.c @@ -30,57 +30,33 @@ #include "scsi_tgt.h" #include "scst_priv.h" -static int scst_do_job_init(void); -static int scst_process_init_cmd(struct scst_cmd *cmd); +static int __scst_init_cmd(struct scst_cmd *cmd); +static int scst_process_active_cmd(struct scst_cmd *cmd, int context); -static int __scst_process_active_cmd(struct scst_cmd *cmd, int context, - int left_locked); - -static void scst_complete_cmd_mgmt(struct scst_cmd *cmd, - struct scst_mgmt_cmd *mcmd); - -/* scst_list_lock assumed to be held */ -static inline int scst_process_active_cmd(struct scst_cmd *cmd, int context, - unsigned long *pflags, int left_locked) -{ - int res; - - TRACE_ENTRY(); - - TRACE_DBG("Moving cmd %p to cmd list", cmd); - list_move_tail(&cmd->cmd_list_entry, &scst_cmd_list); - - /* This is an inline func., so unneeded code will be optimized out */ - if (pflags) - spin_unlock_irqrestore(&scst_list_lock, *pflags); - else - spin_unlock_irq(&scst_list_lock); - - res = __scst_process_active_cmd(cmd, context, left_locked); - - TRACE_EXIT_RES(res); - return res; -} - -/* Called under scst_list_lock and IRQs disabled */ +/* No locks */ static inline void scst_cmd_set_sn(struct scst_cmd *cmd) { /* ToDo: cmd->queue_type */ - /* scst_list_lock is enough to protect that */ - cmd->sn = cmd->tgt_dev->next_sn; - cmd->tgt_dev->next_sn++; + cmd->sn = atomic_inc_return(&cmd->tgt_dev->curr_sn); cmd->no_sn = 0; TRACE(TRACE_DEBUG/*TRACE_SCSI_SERIALIZING*/, "cmd(%p)->sn: %d", cmd, cmd->sn); } -static inline void scst_schedule_tasklet(void) +static inline void scst_schedule_tasklet(struct scst_cmd *cmd) { - struct tasklet_struct *t = &scst_tasklets[smp_processor_id()]; + struct scst_tasklet *t = &scst_tasklets[smp_processor_id()]; + unsigned long flags; - tasklet_schedule(t); + spin_lock_irqsave(&t->tasklet_lock, flags); + TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd, + smp_processor_id()); + list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list); + spin_unlock_irqrestore(&t->tasklet_lock, flags); + + tasklet_schedule(&t->tasklet); } /* @@ -131,17 +107,68 @@ out: return cmd; } -static void scst_setup_to_active(struct scst_cmd *cmd) +static int scst_init_cmd(struct scst_cmd *cmd, int context) { - cmd->state = SCST_CMD_STATE_XMIT_RESP; - TRACE_DBG("Adding cmd %p to active cmd list", cmd); - list_add_tail(&cmd->cmd_list_entry, &scst_active_cmd_list); + int rc; + + TRACE_ENTRY(); + + /* See the comment in scst_do_job_init() */ + if (unlikely(!list_empty(&scst_init_cmd_list))) { + TRACE_MGMT_DBG("%s", "init cmd list busy"); + goto out_redirect; + } + smp_rmb(); + + rc = __scst_init_cmd(cmd); + if (unlikely(rc > 0)) + goto out_redirect; + + /* Small context optimization */ + if (((context == SCST_CONTEXT_TASKLET) || + (context == SCST_CONTEXT_DIRECT_ATOMIC)) && + scst_cmd_is_expected_set(cmd)) { + if (cmd->expected_data_direction == SCST_DATA_WRITE) { + if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC, + &cmd->tgt_dev->tgt_dev_flags)) + context = SCST_CONTEXT_THREAD; + } else { + if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC, + &cmd->tgt_dev->tgt_dev_flags)) + context = SCST_CONTEXT_THREAD; + } + } + +out: + TRACE_EXIT_RES(context); + return context; + +out_redirect: + if (cmd->preprocessing_only) { + /* + * Poor man solution for single threaded targets, where + * blocking receiver at least somtimes means blocking all. + */ + sBUG_ON(context != SCST_CONTEXT_DIRECT); + scst_set_busy(cmd); + cmd->state = SCST_CMD_STATE_XMIT_RESP; + } else { + unsigned long flags; + spin_lock_irqsave(&scst_init_lock, flags); + TRACE_MGMT_DBG("Adding cmd %p to init cmd list", cmd); + list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list); + if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) + scst_init_poll_cnt++; + spin_unlock_irqrestore(&scst_init_lock, flags); + wake_up(&scst_init_cmd_list_waitQ); + context = -1; + } + goto out; } void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context) { - int res = 0; - unsigned long flags = 0; + unsigned long flags; struct scst_session *sess = cmd->sess; TRACE_ENTRY(); @@ -163,11 +190,9 @@ void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context) } #endif - spin_lock_irqsave(&scst_list_lock, flags); + spin_lock_irqsave(&sess->sess_list_lock, flags); - /* Let's make it here, this will save us a lock or atomic */ sess->sess_cmd_count++; - list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list); if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) { @@ -178,21 +203,25 @@ void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context) TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd); list_add_tail(&cmd->cmd_list_entry, &sess->init_deferred_cmd_list); - goto out_unlock_flags; + spin_unlock_irqrestore(&sess->sess_list_lock, flags); + goto out; case SCST_SESS_IPH_FAILED: + spin_unlock_irqrestore(&sess->sess_list_lock, flags); scst_set_busy(cmd); - scst_setup_to_active(cmd); + cmd->state = SCST_CMD_STATE_XMIT_RESP; goto active; default: sBUG(); } } + spin_unlock_irqrestore(&sess->sess_list_lock, flags); + if (unlikely(cmd->lun == (lun_t)-1)) { PRINT_ERROR_PR("Wrong LUN %d, finishing cmd", -1); scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_lun_not_supported)); - scst_setup_to_active(cmd); + cmd->state = SCST_CMD_STATE_XMIT_RESP; goto active; } @@ -200,74 +229,46 @@ void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context) PRINT_ERROR_PR("Wrong CDB len %d, finishing cmd", 0); scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode)); - scst_setup_to_active(cmd); + cmd->state = SCST_CMD_STATE_XMIT_RESP; goto active; } - TRACE_DBG("Adding cmd %p to init cmd list", cmd); - list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list); - cmd->state = SCST_CMD_STATE_INIT; - - switch (pref_context) { - case SCST_CONTEXT_TASKLET: - scst_schedule_tasklet(); - goto out_unlock_flags; - - case SCST_CONTEXT_DIRECT: - case SCST_CONTEXT_DIRECT_ATOMIC: - if (cmd->no_sn) - res = scst_process_init_cmd(cmd); - else - res = scst_do_job_init(); - if (unlikely(res > 0)) - goto out_unlock_flags; - break; - - case SCST_CONTEXT_THREAD: - goto out_thread_unlock_flags; - - default: - PRINT_ERROR_PR("Context %x is undefined, using the thread one", - pref_context); - goto out_thread_unlock_flags; - } + /* cmd must be inited here to keep the order */ + pref_context = scst_init_cmd(cmd, pref_context); + if (unlikely(pref_context < 0)) + goto out; active: - /* Here cmd must be in active cmd list */ + /* Here cmd must not be in any cmd list, no locks */ switch (pref_context) { case SCST_CONTEXT_TASKLET: - scst_schedule_tasklet(); - goto out_unlock_flags; + scst_schedule_tasklet(cmd); + break; case SCST_CONTEXT_DIRECT: case SCST_CONTEXT_DIRECT_ATOMIC: - scst_process_active_cmd(cmd, pref_context, &flags, 0); + scst_process_active_cmd(cmd, pref_context); /* For *NEED_THREAD wake_up() is already done */ break; - case SCST_CONTEXT_THREAD: - goto out_thread_unlock_flags; - default: PRINT_ERROR_PR("Context %x is undefined, using the thread one", pref_context); - goto out_thread_unlock_flags; + /* go through */ + case SCST_CONTEXT_THREAD: + spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags); + TRACE_DBG("Adding cmd %p to active cmd list", cmd); + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags); + break; } out: TRACE_EXIT(); return; - -out_unlock_flags: - spin_unlock_irqrestore(&scst_list_lock, flags); - goto out; - -out_thread_unlock_flags: - cmd->non_atomic_only = 1; - spin_unlock_irqrestore(&scst_list_lock, flags); - wake_up(&scst_list_waitQ); - goto out; } static int scst_parse_cmd(struct scst_cmd *cmd) @@ -283,12 +284,6 @@ static int scst_parse_cmd(struct scst_cmd *cmd) TRACE_ENTRY(); - if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) { - TRACE_DBG("ABORTED set, returning ABORTED " - "for cmd %p", cmd); - goto out_xmit; - } - if (atomic && !dev->handler->parse_atomic) { TRACE_DBG("Dev handler %s parse() can not be " "called in atomic context, rescheduling to the thread", @@ -646,6 +641,14 @@ prep_done: goto out; } + if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) { + TRACE_DBG("ABORTED set, returning ABORTED for " + "cmd %p", cmd); + cmd->state = SCST_CMD_STATE_DEV_DONE; + res = SCST_CMD_STATE_RES_CONT_SAME; + goto out; + } + res = SCST_CMD_STATE_RES_CONT_NEXT; cmd->state = SCST_CMD_STATE_PREPROCESS_DONE; @@ -686,7 +689,6 @@ void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context) TRACE_DBG("Preferred context: %d", pref_context); TRACE_DBG("tag=%d, status=%#x", scst_cmd_get_tag(cmd), status); - cmd->non_atomic_only = 0; #ifdef EXTRACHECKS if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) || @@ -709,25 +711,21 @@ void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context) cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV; break; } - if (cmd->no_sn) { - unsigned long flags; - int rc; - spin_lock_irqsave(&scst_list_lock, flags); - /* Necessary to keep the command's order */ - rc = scst_do_job_init(); - if (unlikely(rc > 0)) { - TRACE_DBG("Adding cmd %p to init cmd list", - cmd); - list_add_tail(&cmd->cmd_list_entry, - &scst_init_cmd_list); - spin_unlock_irqrestore(&scst_list_lock, flags); - goto out; - } + if (cmd->no_sn) scst_cmd_set_sn(cmd); - spin_unlock_irqrestore(&scst_list_lock, flags); + /* Small context optimization */ + if ((pref_context == SCST_CONTEXT_TASKLET) || + (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) { + if (cmd->data_direction == SCST_DATA_WRITE) { + if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC, + &cmd->tgt_dev->tgt_dev_flags)) + pref_context = SCST_CONTEXT_THREAD; + } else { + if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC, + &cmd->tgt_dev->tgt_dev_flags)) + pref_context = SCST_CONTEXT_THREAD; + } } - if (tm_dbg_check_cmd(cmd) != 0) - goto out; break; case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET: @@ -744,7 +742,7 @@ void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context) break; default: - PRINT_ERROR_PR("scst_rx_data() received unknown status %x", + PRINT_ERROR_PR("%s() received unknown status %x", __func__, status); cmd->state = SCST_CMD_STATE_DEV_DONE; break; @@ -752,7 +750,6 @@ void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context) scst_proccess_redirect_cmd(cmd, pref_context, 1); -out: TRACE_EXIT(); return; } @@ -782,11 +779,8 @@ static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds) goto out_unlock_tgt; } - TRACE(TRACE_RETRY, "Moving cmd %p to retry cmd list", cmd); - /* IRQ already off */ - spin_lock(&scst_list_lock); - list_move_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list); - spin_unlock(&scst_list_lock); + TRACE(TRACE_RETRY, "Adding cmd %p to retry cmd list", cmd); + list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list); if (!tgt->retry_timer_active) { tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT; @@ -808,10 +802,8 @@ static int scst_rdy_to_xfer(struct scst_cmd *cmd) TRACE_ENTRY(); - if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) - { - TRACE_DBG("ABORTED set, returning ABORTED for " - "cmd %p", cmd); + if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) { + TRACE_DBG("ABORTED set, aborting cmd %p", cmd); goto out_dev_done; } @@ -893,11 +885,11 @@ out_dev_done: goto out; } +/* No locks, but might be in IRQ */ void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context, int check_retries) { unsigned long flags; - int rc; TRACE_ENTRY(); @@ -907,11 +899,8 @@ void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context, case SCST_CONTEXT_DIRECT: case SCST_CONTEXT_DIRECT_ATOMIC: if (check_retries) - scst_check_retries(cmd->tgt, 0); - cmd->non_atomic_only = 0; - rc = __scst_process_active_cmd(cmd, context, 0); - if (rc == SCST_CMD_STATE_RES_NEED_THREAD) - goto out_thread; + scst_check_retries(cmd->tgt); + scst_process_active_cmd(cmd, context); break; default: @@ -920,32 +909,23 @@ void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context, /* go through */ case SCST_CONTEXT_THREAD: if (check_retries) - scst_check_retries(cmd->tgt, 1); - goto out_thread; + scst_check_retries(cmd->tgt); + spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags); + TRACE_DBG("Adding cmd %p to active cmd list", cmd); + list_add_tail(&cmd->cmd_list_entry, &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags); + break; case SCST_CONTEXT_TASKLET: if (check_retries) - scst_check_retries(cmd->tgt, 1); - cmd->non_atomic_only = 0; - spin_lock_irqsave(&scst_list_lock, flags); - TRACE_DBG("Moving cmd %p to active cmd list", cmd); - list_move_tail(&cmd->cmd_list_entry, &scst_active_cmd_list); - spin_unlock_irqrestore(&scst_list_lock, flags); - scst_schedule_tasklet(); + scst_check_retries(cmd->tgt); + scst_schedule_tasklet(cmd); break; } -out: + TRACE_EXIT(); return; - -out_thread: - cmd->non_atomic_only = 1; - spin_lock_irqsave(&scst_list_lock, flags); - TRACE_DBG("Moving cmd %p to active cmd list", cmd); - list_move_tail(&cmd->cmd_list_entry, &scst_active_cmd_list); - spin_unlock_irqrestore(&scst_list_lock, flags); - wake_up(&scst_list_waitQ); - goto out; } void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context) @@ -954,7 +934,6 @@ void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context) TRACE_DBG("Preferred context: %d", pref_context); TRACE(TRACE_SCSI, "tag=%d status=%#x", scst_cmd_get_tag(cmd), status); - cmd->non_atomic_only = 0; #ifdef EXTRACHECKS if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) || @@ -970,6 +949,13 @@ void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context) switch (status) { case SCST_RX_STATUS_SUCCESS: cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV; + /* Small context optimization */ + if ((pref_context == SCST_CONTEXT_TASKLET) || + (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) { + if ( !test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC, + &cmd->tgt_dev->tgt_dev_flags)) + pref_context = SCST_CONTEXT_THREAD; + } break; case SCST_RX_STATUS_ERROR_SENSE_SET: @@ -1212,6 +1198,19 @@ next: return; } +/* For small context optimization */ +static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd, + int context) +{ + if ((context == SCST_CONTEXT_TASKLET) || + (context == SCST_CONTEXT_DIRECT_ATOMIC)) { + if ( !test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC, + &cmd->tgt_dev->tgt_dev_flags)) + context = SCST_CONTEXT_THREAD; + } + return context; +} + #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd, struct scsi_request **req) @@ -1259,9 +1258,9 @@ static void scst_cmd_done(struct scsi_cmnd *scsi_cmd) scst_release_request(cmd); cmd->state = next_state; - cmd->non_atomic_only = 0; - scst_proccess_redirect_cmd(cmd, scst_get_context(), 0); + scst_proccess_redirect_cmd(cmd, + scst_optimize_post_exec_context(cmd, scst_get_context()), 0); out: TRACE_EXIT(); @@ -1286,9 +1285,9 @@ static void scst_cmd_done(void *data, char *sense, int result, int resid) &next_state); cmd->state = next_state; - cmd->non_atomic_only = 0; - scst_proccess_redirect_cmd(cmd, scst_get_context(), 0); + scst_proccess_redirect_cmd(cmd, + scst_optimize_post_exec_context(cmd, scst_get_context()), 0); out: TRACE_EXIT(); @@ -1346,9 +1345,9 @@ static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state) scst_check_sense(cmd, NULL, 0, &next_state); cmd->state = next_state; - cmd->non_atomic_only = 0; - scst_proccess_redirect_cmd(cmd, scst_get_context(), 0); + scst_proccess_redirect_cmd(cmd, + scst_optimize_post_exec_context(cmd, scst_get_context()), 0); TRACE_EXIT(); return; @@ -1884,7 +1883,7 @@ static int scst_send_to_midlev(struct scst_cmd *cmd) res = SCST_CMD_STATE_RES_CONT_NEXT; - if (atomic && dev->handler->exec && !dev->handler->exec_atomic) { + if (atomic && !dev->handler->exec_atomic) { TRACE_DBG("Dev handler %s exec() can not be " "called in atomic context, rescheduling to the thread", dev->handler->name); @@ -1895,7 +1894,7 @@ static int scst_send_to_midlev(struct scst_cmd *cmd) if (unlikely(scst_inc_on_dev_cmd(cmd) != 0)) goto out; - scst_inc_cmd_count(); /* protect dev & tgt_dev */ + __scst_get(0); /* protect dev & tgt_dev */ if (unlikely(cmd->internal) || unlikely(cmd->retry)) { rc = scst_do_send_to_midlev(cmd); @@ -1923,10 +1922,20 @@ static int scst_send_to_midlev(struct scst_cmd *cmd) expected_sn = tgt_dev->expected_sn; if (cmd->sn != expected_sn) { scst_dec_on_dev_cmd(cmd); - TRACE(TRACE_SCSI_SERIALIZING, "Delaying cmd %p (sn=%d, " - "expected_sn=%d)", cmd, cmd->sn, expected_sn); - list_add_tail(&cmd->sn_cmd_list_entry, - &tgt_dev->deferred_cmd_list); + if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) { + /* Necessary to allow aborting out of sn cmds */ + TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %d)", + cmd, cmd->tag); + tgt_dev->def_cmd_count--; + cmd->state = SCST_CMD_STATE_DEV_DONE; + res = SCST_CMD_STATE_RES_CONT_SAME; + } else { + TRACE(TRACE_SCSI_SERIALIZING, "Delaying cmd %p " + "(sn=%d, expected_sn=%d)", cmd, + cmd->sn, expected_sn); + list_add_tail(&cmd->sn_cmd_list_entry, + &tgt_dev->deferred_cmd_list); + } spin_unlock_bh(&tgt_dev->sn_lock); /* !! At this point cmd can be already freed !! */ goto out_dec_cmd_count; @@ -1967,7 +1976,7 @@ out_unplug: generic_unplug_device(dev->scsi_dev->request_queue); out_dec_cmd_count: - scst_dec_cmd_count(); + __scst_put(); /* !! At this point sess, dev and tgt_dev can be already freed !! */ out: @@ -1990,7 +1999,7 @@ static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres) "without sense data (opcode 0x%x), issuing " "REQUEST SENSE", cmd->cdb[0]); rc = scst_prepare_request_sense(cmd); - if (res > 0) { + if (rc > 0) { *pres = rc; res = 1; goto out; @@ -2109,8 +2118,7 @@ static int scst_dev_done(struct scst_cmd *cmd) TRACE_ENTRY(); - if (atomic && !cmd->dev->handler->dev_done_atomic && - cmd->dev->handler->dev_done) + if (atomic && !cmd->dev->handler->dev_done_atomic) { TRACE_DBG("Dev handler %s dev_done() can not be " "called in atomic context, rescheduling to the thread", @@ -2198,7 +2206,7 @@ static int scst_xmit_response(struct scst_cmd *cmd) { TRACE(TRACE_SCSI_SERIALIZING, "cmd %p was not sent to mid-lev (sn %d)", cmd, cmd->sn); - scst_inc_expected_sn_unblock(cmd->tgt_dev, cmd, 0); + scst_inc_expected_sn_unblock(cmd->tgt_dev, cmd); cmd->sent_to_midlev = 1; } @@ -2346,24 +2354,12 @@ static int scst_finish_cmd(struct scst_cmd *cmd) spin_unlock_bh(&scst_cmd_mem_lock); } - spin_lock_irq(&scst_list_lock); - - TRACE_DBG("Deleting cmd %p from cmd list", cmd); - list_del(&cmd->cmd_list_entry); - - if (cmd->mgmt_cmnd) - scst_complete_cmd_mgmt(cmd, cmd->mgmt_cmnd); - - if (likely(cmd->tgt_dev != NULL)) - cmd->tgt_dev->cmd_count--; - + spin_lock_irq(&cmd->sess->sess_list_lock); cmd->sess->sess_cmd_count--; - list_del(&cmd->search_cmd_list_entry); + spin_unlock_irq(&cmd->sess->sess_list_lock); - spin_unlock_irq(&scst_list_lock); - - scst_free_cmd(cmd); + scst_cmd_put(cmd); res = SCST_CMD_STATE_RES_CONT_NEXT; @@ -2374,7 +2370,9 @@ static int scst_finish_cmd(struct scst_cmd *cmd) /* * Returns 0 on success, > 0 when we need to wait for unblock, * < 0 if there is no device (lun) or device type handler. - * Called under scst_list_lock and IRQs disabled + * + * No locks, but might be on IRQ, protection is done by the + * suspended activity. */ static int scst_translate_lun(struct scst_cmd *cmd) { @@ -2383,7 +2381,7 @@ static int scst_translate_lun(struct scst_cmd *cmd) TRACE_ENTRY(); - scst_inc_cmd_count(); + __scst_get(1); if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) { res = -1; @@ -2403,16 +2401,16 @@ static int scst_translate_lun(struct scst_cmd *cmd) } if (cmd->state == SCST_CMD_STATE_REINIT) { - cmd->tgt_dev_saved->cmd_count--; + atomic_dec(&cmd->tgt_dev_saved->cmd_count); TRACE(TRACE_SCSI_SERIALIZING, "SCST_CMD_STATE_REINIT: " "incrementing expected_sn on tgt_dev_saved %p", cmd->tgt_dev_saved); scst_inc_expected_sn_unblock( - cmd->tgt_dev_saved, cmd, 1); + cmd->tgt_dev_saved, cmd); } + cmd->cmd_lists = tgt_dev->p_cmd_lists; cmd->tgt_dev = tgt_dev; - tgt_dev->cmd_count++; cmd->dev = tgt_dev->acg_dev->dev; res = 0; @@ -2420,19 +2418,13 @@ static int scst_translate_lun(struct scst_cmd *cmd) } } if (res != 0) { - TRACE_DBG("tgt_dev for lun %Ld not found, command to " + TRACE(TRACE_MINOR, "tgt_dev for lun %Ld not found, command to " "unexisting LU?", (uint64_t)cmd->lun); - scst_dec_cmd_count(); + __scst_put(); } } else { - if ( !cmd->sess->waiting) { - TRACE_DBG("Adding session %p to scst_dev_wait_sess_list", - cmd->sess); - list_add_tail(&cmd->sess->dev_wait_sess_list_entry, - &scst_dev_wait_sess_list); - cmd->sess->waiting = 1; - } - scst_dec_cmd_count(); + TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping"); + __scst_put(); res = 1; } @@ -2440,22 +2432,24 @@ static int scst_translate_lun(struct scst_cmd *cmd) return res; } -/* Called under scst_list_lock and IRQs disabled */ -static int scst_process_init_cmd(struct scst_cmd *cmd) +/* + * No locks, but might be on IRQ + * + * Returns 0 on success, > 0 when we need to wait for unblock, + * < 0 if there is no device (lun) or device type handler. + */ +static int __scst_init_cmd(struct scst_cmd *cmd) { int res = 0; TRACE_ENTRY(); - if (unlikely(cmd->tgt_dev)) { - scst_cmd_set_sn(cmd); - goto out_move; - } - res = scst_translate_lun(cmd); if (likely(res == 0)) { + int cnt; cmd->state = SCST_CMD_STATE_DEV_PARSE; - if (cmd->tgt_dev->cmd_count > SCST_MAX_DEVICE_COMMANDS) { + cnt = atomic_inc_return(&cmd->tgt_dev->cmd_count); + if (unlikely(cnt > SCST_MAX_DEVICE_COMMANDS)) { TRACE(TRACE_RETRY, "Too many pending commands in " "session, returning BUSY to initiator \"%s\"", (cmd->sess->initiator_name[0] == '\0') ? @@ -2472,47 +2466,127 @@ static int scst_process_init_cmd(struct scst_cmd *cmd) } else goto out; -out_move: - TRACE_DBG("Moving cmd %p to active cmd list", cmd); - list_move_tail(&cmd->cmd_list_entry, &scst_active_cmd_list); - out: TRACE_EXIT_RES(res); return res; } -/* - * Called under scst_list_lock and IRQs disabled - * We don't drop it anywhere inside, because command execution - * have to be serialized, i.e. commands must be executed in order - * of their arrival, and we set this order inside scst_translate_lun(). - */ -static int scst_do_job_init(void) +/* Called under scst_init_lock and IRQs disabled */ +static void scst_do_job_init(void) { - int res = 0; + struct scst_cmd *cmd; + int susp; TRACE_ENTRY(); - if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) { - while (!list_empty(&scst_init_cmd_list)) { - struct scst_cmd *cmd = list_entry( - scst_init_cmd_list.next, typeof(*cmd), - cmd_list_entry); - res = scst_process_init_cmd(cmd); - if (res > 0) - break; - /* For DIRECT context the cmd is always the last */ - } - } else - res = 1; +restart: + susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags); + if (scst_init_poll_cnt > 0) + scst_init_poll_cnt--; - TRACE_EXIT_RES(res); + list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) { + int rc; + if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) + continue; + if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) { + spin_unlock_irq(&scst_init_lock); + rc = __scst_init_cmd(cmd); + spin_lock_irq(&scst_init_lock); + if (rc > 0) { + TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting"); + goto restart; + } + } else { + TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %d)", + cmd, cmd->tag); + cmd->state = SCST_CMD_STATE_XMIT_RESP; + } + + /* + * Deleting cmd from init cmd list after __scst_init_cmd() + * is necessary to keep the check in scst_init_cmd() correct + * to preserve the commands order. + * + * We don't care about the race, when init cmd list is empty + * and one command detected that it just was not empty, so + * it's inserting to it, but another command at the same time + * seeing init cmd list empty and goes directly, because it + * could affect only commands from the same initiator to the + * same tgt_dev, but init_cmd_done() doesn't guarantee the order + * in case of simultaneous such calls anyway. + */ + TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd); + list_del(&cmd->cmd_list_entry); + spin_unlock(&scst_init_lock); + + spin_lock(&cmd->cmd_lists->cmd_list_lock); + TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd); + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock(&cmd->cmd_lists->cmd_list_lock); + + spin_lock(&scst_init_lock); + goto restart; + } + + if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) + goto restart; + + TRACE_EXIT(); + return; +} + +static inline int test_init_cmd_list(void) +{ + int res = (!list_empty(&scst_init_cmd_list) && + !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) || + unlikely(kthread_should_stop()) || + (scst_init_poll_cnt > 0); return res; } +int scst_init_cmd_thread(void *arg) +{ + TRACE_ENTRY(); + + current->flags |= PF_NOFREEZE; + + spin_lock_irq(&scst_init_lock); + while(!kthread_should_stop()) { + wait_queue_t wait; + init_waitqueue_entry(&wait, current); + + if (!test_init_cmd_list()) { + add_wait_queue_exclusive(&scst_init_cmd_list_waitQ, + &wait); + for (;;) { + set_current_state(TASK_INTERRUPTIBLE); + if (test_init_cmd_list()) + break; + spin_unlock_irq(&scst_init_lock); + schedule(); + spin_lock_irq(&scst_init_lock); + } + set_current_state(TASK_RUNNING); + remove_wait_queue(&scst_init_cmd_list_waitQ, &wait); + } + scst_do_job_init(); + } + spin_unlock_irq(&scst_init_lock); + + /* + * If kthread_should_stop() is true, we are guaranteed to be + * on the module unload, so scst_init_cmd_list must be empty. + */ + sBUG_ON(!list_empty(&scst_init_cmd_list)); + + TRACE_EXIT(); + return 0; +} + /* Called with no locks held */ -static int __scst_process_active_cmd(struct scst_cmd *cmd, int context, - int left_locked) +static int scst_process_active_cmd(struct scst_cmd *cmd, int context) { int res; @@ -2520,10 +2594,9 @@ static int __scst_process_active_cmd(struct scst_cmd *cmd, int context, EXTRACHECKS_BUG_ON(in_irq()); - cmd->atomic = ((context & ~SCST_PROCESSIBLE_ENV) == - SCST_CONTEXT_DIRECT_ATOMIC); - cmd->processible_env = (context & SCST_PROCESSIBLE_ENV) != 0; + cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC); +restart: do { switch (cmd->state) { case SCST_CMD_STATE_DEV_PARSE: @@ -2539,6 +2612,13 @@ static int __scst_process_active_cmd(struct scst_cmd *cmd, int context, break; case SCST_CMD_STATE_SEND_TO_MIDLEV: + if (tm_dbg_check_cmd(cmd) != 0) { + res = SCST_CMD_STATE_RES_CONT_NEXT; + TRACE_MGMT_DBG("Skipping cmd %p (tag %d), " + "because of TM DBG delay", cmd, + cmd->tag); + break; + } res = scst_send_to_midlev(cmd); /* !! At this point cmd, sess & tgt_dev can be already freed !! */ break; @@ -2565,11 +2645,9 @@ static int __scst_process_active_cmd(struct scst_cmd *cmd, int context, } while(res == SCST_CMD_STATE_RES_CONT_SAME); if (res == SCST_CMD_STATE_RES_CONT_NEXT) { - if (left_locked) - spin_lock_irq(&scst_list_lock); + /* None */ } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) { - spin_lock_irq(&scst_list_lock); - + spin_lock_irq(&cmd->cmd_lists->cmd_list_lock); switch (cmd->state) { case SCST_CMD_STATE_DEV_PARSE: case SCST_CMD_STATE_PREPARE_SPACE: @@ -2578,8 +2656,9 @@ static int __scst_process_active_cmd(struct scst_cmd *cmd, int context, case SCST_CMD_STATE_DEV_DONE: case SCST_CMD_STATE_XMIT_RESP: case SCST_CMD_STATE_FINISHED: - TRACE_DBG("Moving cmd %p to active cmd list", cmd); - list_move(&cmd->cmd_list_entry, &scst_active_cmd_list); + TRACE_DBG("Adding cmd %p to head of active cmd list", cmd); + list_add(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); break; #ifdef EXTRACHECKS /* not very valid commands */ @@ -2588,25 +2667,31 @@ static int __scst_process_active_cmd(struct scst_cmd *cmd, int context, PRINT_ERROR_PR("cmd %p is in state %d, not putting on " "useful list (left on scst cmd list)", cmd, cmd->state); - spin_unlock_irq(&scst_list_lock); + spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock); sBUG(); - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&cmd->cmd_lists->cmd_list_lock); break; #endif default: break; } - cmd->non_atomic_only = 1; - if (!left_locked) - spin_unlock_irq(&scst_list_lock); - wake_up(&scst_list_waitQ); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock); } else if (res == SCST_CMD_STATE_RES_RESTART) { if (cmd->state == SCST_CMD_STATE_REINIT) { - spin_lock_irq(&scst_list_lock); - TRACE_DBG("Moving cmd %p to head of init cmd list", cmd); - list_move(&cmd->cmd_list_entry, &scst_init_cmd_list); - if (!left_locked) - spin_unlock_irq(&scst_list_lock); + int rc; + rc = scst_init_cmd(cmd, context); + if (rc == context) + goto restart; + else if (rc == SCST_CONTEXT_THREAD) { + spin_lock_irq(&cmd->cmd_lists->cmd_list_lock); + TRACE_DBG("Adding cmd %p to active cmd list", + cmd); + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock); + } else if (rc >= 0) + sBUG(); } else sBUG(); } else @@ -2616,54 +2701,40 @@ static int __scst_process_active_cmd(struct scst_cmd *cmd, int context, return res; } -/* Called under scst_list_lock and IRQs disabled */ -static void scst_do_job_active(int context) +/* Called under cmd_list_lock and IRQs disabled */ +static void scst_do_job_active(struct list_head *cmd_list, + spinlock_t *cmd_list_lock, int context) { - int res; - struct scst_cmd *cmd; - int atomic = ((context & ~SCST_PROCESSIBLE_ENV) == - SCST_CONTEXT_DIRECT_ATOMIC); - TRACE_ENTRY(); #ifdef EXTRACHECKS - { - int c = (context & ~SCST_PROCESSIBLE_ENV); - WARN_ON((c != SCST_CONTEXT_DIRECT_ATOMIC) && - (c != SCST_CONTEXT_DIRECT)); - } + WARN_ON((context != SCST_CONTEXT_DIRECT_ATOMIC) && + (context != SCST_CONTEXT_DIRECT)); #endif - tm_dbg_check_released_cmds(); - -restart: - list_for_each_entry(cmd, &scst_active_cmd_list, cmd_list_entry) { - if (atomic && cmd->non_atomic_only) { - TRACE(TRACE_DEBUG, "Skipping non-atomic cmd %p", cmd); - continue; - } - if (tm_dbg_check_cmd(cmd) != 0) - goto restart; - res = scst_process_active_cmd(cmd, context, NULL, 1); - if (res == SCST_CMD_STATE_RES_CONT_NEXT) { - goto restart; - } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) { - goto restart; - } else if (res == SCST_CMD_STATE_RES_RESTART) { - break; - } else + while (!list_empty(cmd_list)) { + int rc; + struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd), + cmd_list_entry); + TRACE_DBG("Deleting cmd %p from active cmd list", cmd); + list_del(&cmd->cmd_list_entry); + spin_unlock_irq(cmd_list_lock); + rc = scst_process_active_cmd(cmd, context); + if (unlikely((rc != SCST_CMD_STATE_RES_CONT_NEXT) && + (rc != SCST_CMD_STATE_RES_NEED_THREAD) && + (rc != SCST_CMD_STATE_RES_RESTART))) { sBUG(); + } + spin_lock_irq(cmd_list_lock); } TRACE_EXIT(); return; } -static inline int test_cmd_lists(void) +static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists) { - int res = !list_empty(&scst_active_cmd_list) || - (!list_empty(&scst_init_cmd_list) && - !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) || + int res = !list_empty(&p_cmd_lists->active_cmd_list) || unlikely(kthread_should_stop()) || tm_dbg_is_release(); return res; @@ -2671,44 +2742,52 @@ static inline int test_cmd_lists(void) int scst_cmd_thread(void *arg) { + struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists*)arg; + TRACE_ENTRY(); set_user_nice(current, 10); current->flags |= PF_NOFREEZE; - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&p_cmd_lists->cmd_list_lock); while (!kthread_should_stop()) { wait_queue_t wait; init_waitqueue_entry(&wait, current); - if (!test_cmd_lists()) { - add_wait_queue_exclusive(&scst_list_waitQ, &wait); + if (!test_cmd_lists(p_cmd_lists)) { + add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ, + &wait); for (;;) { set_current_state(TASK_INTERRUPTIBLE); - if (test_cmd_lists()) + if (test_cmd_lists(p_cmd_lists)) break; - spin_unlock_irq(&scst_list_lock); + spin_unlock_irq(&p_cmd_lists->cmd_list_lock); schedule(); - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&p_cmd_lists->cmd_list_lock); } set_current_state(TASK_RUNNING); - remove_wait_queue(&scst_list_waitQ, &wait); + remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait); } - scst_do_job_init(); - scst_do_job_active(SCST_CONTEXT_DIRECT|SCST_PROCESSIBLE_ENV); + if (tm_dbg_is_release()) { + spin_unlock_irq(&p_cmd_lists->cmd_list_lock); + tm_dbg_check_released_cmds(); + spin_lock_irq(&p_cmd_lists->cmd_list_lock); + } + + scst_do_job_active(&p_cmd_lists->active_cmd_list, + &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT); } - spin_unlock_irq(&scst_list_lock); + spin_unlock_irq(&p_cmd_lists->cmd_list_lock); /* * If kthread_should_stop() is true, we are guaranteed to be either * on the module unload, or there must be at least one other thread to * process the commands lists. */ - sBUG_ON((scst_threads_info.nr_cmd_threads == 1) && - (!list_empty(&scst_cmd_list) || - !list_empty(&scst_active_cmd_list) || - !list_empty(&scst_init_cmd_list))); +//!!ToDo sBUG_ON((scst_threads_info.nr_cmd_threads == 1) && +// (!list_empty(&scst_cmd_lists) || +// !list_empty(&scst_active_cmd_lists))); TRACE_EXIT(); return 0; @@ -2716,14 +2795,14 @@ int scst_cmd_thread(void *arg) void scst_cmd_tasklet(long p) { + struct scst_tasklet *t = (struct scst_tasklet*)p; + TRACE_ENTRY(); - spin_lock_irq(&scst_list_lock); - - scst_do_job_init(); - scst_do_job_active(SCST_CONTEXT_DIRECT_ATOMIC|SCST_PROCESSIBLE_ENV); - - spin_unlock_irq(&scst_list_lock); + spin_lock_irq(&t->tasklet_lock); + scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock, + SCST_CONTEXT_DIRECT_ATOMIC); + spin_unlock_irq(&t->tasklet_lock); TRACE_EXIT(); return; @@ -2731,7 +2810,8 @@ void scst_cmd_tasklet(long p) /* * Returns 0 on success, < 0 if there is no device handler or - * > 0 if SCST_FLAG_SUSPENDED set. + * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not. + * No locks, protection is done by the suspended activity. */ static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd) { @@ -2743,44 +2823,41 @@ static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd) TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd, (uint64_t)mcmd->lun); - spin_lock_irq(&scst_list_lock); - scst_inc_cmd_count(); - if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) { - list_for_each_entry(tgt_dev, &mcmd->sess->sess_tgt_dev_list, - sess_tgt_dev_list_entry) - { - if (tgt_dev->acg_dev->lun == mcmd->lun) { - TRACE_DBG("tgt_dev %p found", tgt_dev); - mcmd->mcmd_tgt_dev = tgt_dev; - res = 0; - break; - } - } - if (mcmd->mcmd_tgt_dev == NULL) - scst_dec_cmd_count(); - } else { - if ( !mcmd->sess->waiting) { - TRACE_DBG("Adding session %p to scst_dev_wait_sess_list", - mcmd->sess); - list_add_tail(&mcmd->sess->dev_wait_sess_list_entry, - &scst_dev_wait_sess_list); - mcmd->sess->waiting = 1; - } - scst_dec_cmd_count(); - res = 1; - } - spin_unlock_irq(&scst_list_lock); + __scst_get(1); + if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) && + !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) { + TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping"); + __scst_put(); + res = 1; + goto out; + } + + list_for_each_entry(tgt_dev, &mcmd->sess->sess_tgt_dev_list, + sess_tgt_dev_list_entry) + { + if (tgt_dev->acg_dev->lun == mcmd->lun) { + TRACE_DBG("tgt_dev %p found", tgt_dev); + mcmd->mcmd_tgt_dev = tgt_dev; + res = 0; + break; + } + } + if (mcmd->mcmd_tgt_dev == NULL) + __scst_put(); + +out: TRACE_EXIT_HRES(res); return res; } -/* Called under scst_list_lock and IRQ off */ -static void scst_complete_cmd_mgmt(struct scst_cmd *cmd, - struct scst_mgmt_cmd *mcmd) +/* No locks */ +void scst_complete_cmd_mgmt(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd) { TRACE_ENTRY(); + spin_lock_irq(&scst_mcmd_lock); + TRACE_MGMT_DBG("cmd %p completed (tag %d, mcmd %p, " "mcmd->cmd_wait_count %d)", cmd, cmd->tag, mcmd, mcmd->cmd_wait_count); @@ -2794,42 +2871,44 @@ static void scst_complete_cmd_mgmt(struct scst_cmd *cmd, if (mcmd->cmd_wait_count > 0) { TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, skipping", mcmd->cmd_wait_count); - goto out; + goto out_unlock; } mcmd->state = SCST_MGMT_CMD_STATE_DONE; if (mcmd->completed) { - TRACE_MGMT_DBG("Moving mgmt cmd %p to active mgmt cmd list", + TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd list", mcmd); - list_move_tail(&mcmd->mgmt_cmd_list_entry, + list_add_tail(&mcmd->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list); } + spin_unlock_irq(&scst_mcmd_lock); + wake_up(&scst_mgmt_cmd_list_waitQ); out: TRACE_EXIT(); return; + +out_unlock: + spin_unlock_irq(&scst_mcmd_lock); + goto out; } static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd, struct scst_tgt_dev *tgt_dev, int set_status) { int res = SCST_DEV_TM_NOT_COMPLETED; - if (tgt_dev->acg_dev->dev->handler->task_mgmt_fn) { - int irq = irqs_disabled(); + struct scst_dev_type *h = tgt_dev->acg_dev->dev->handler; + + if (h->task_mgmt_fn) { TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)", - tgt_dev->acg_dev->dev->handler->name, mcmd->fn); + h->name, mcmd->fn); EXTRACHECKS_BUG_ON(in_irq()); - if (!irq) - local_bh_disable(); - res = tgt_dev->acg_dev->dev->handler->task_mgmt_fn(mcmd, - tgt_dev); - if (!irq) - local_bh_enable(); + res = h->task_mgmt_fn(mcmd, tgt_dev); TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn() returned %d", - tgt_dev->acg_dev->dev->handler->name, res); + h->name, res); if (set_status && (res != SCST_DEV_TM_NOT_COMPLETED)) mcmd->status = res; } @@ -2849,8 +2928,7 @@ static inline int scst_is_strict_mgmt_fn(int mgmt_fn) } /* - * Called under scst_list_lock and IRQ off (to protect cmd - * from being destroyed) + BHs also off + * Might be called under sess_list_lock and IRQ off + BHs also off * Returns -1 if command is being executed (ABORT failed), 0 otherwise */ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, @@ -2867,8 +2945,18 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, set_bit(SCST_CMD_ABORTED, &cmd->cmd_flags); smp_mb__after_set_bit(); - if (call_dev_task_mgmt_fn && cmd->tgt_dev) + if (cmd->tgt_dev == NULL) { + unsigned long flags; + spin_lock_irqsave(&scst_init_lock, flags); + scst_init_poll_cnt++; + spin_unlock_irqrestore(&scst_init_lock, flags); + wake_up(&scst_init_cmd_list_waitQ); + } + + if (call_dev_task_mgmt_fn && (cmd->tgt_dev != NULL)) { + EXTRACHECKS_BUG_ON(irqs_disabled()); scst_call_dev_task_mgmt_fn(mcmd, cmd->tgt_dev, 1); + } if (mcmd) { int defer; @@ -2884,6 +2972,7 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, } if (defer) { + unsigned long flags; /* * Delay the response until the command's finish in * order to guarantee that "no further responses from @@ -2902,7 +2991,10 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, } #endif sBUG_ON(cmd->mgmt_cmnd); + spin_lock_irqsave(&scst_mcmd_lock, flags); mcmd->cmd_wait_count++; + spin_unlock_irqrestore(&scst_mcmd_lock, flags); + /* cmd can't die here or sess_list_lock already taken */ cmd->mgmt_cmnd = mcmd; } } @@ -2913,10 +3005,11 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, return; } -/* Called under scst_list_lock and IRQ off */ +/* No locks */ static int scst_set_mcmd_next_state(struct scst_mgmt_cmd *mcmd) { int res; + spin_lock_irq(&scst_mcmd_lock); if (mcmd->cmd_wait_count != 0) { TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, preparing to " "wait", mcmd->cmd_wait_count); @@ -2927,13 +3020,30 @@ static int scst_set_mcmd_next_state(struct scst_mgmt_cmd *mcmd) res = 0; } mcmd->completed = 1; + spin_unlock_irq(&scst_mcmd_lock); + return res; +} + +static int __scst_check_unblock_aborted_cmd(struct scst_cmd *cmd) +{ + int res; + if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) { + TRACE_MGMT_DBG("Adding aborted blocked cmd %p to active cmd " + "list", cmd); + spin_lock(&cmd->cmd_lists->cmd_list_lock); + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock(&cmd->cmd_lists->cmd_list_lock); + res = 1; + } else + res = 0; return res; } static void scst_unblock_aborted_cmds(int scst_mutex_held) { struct scst_device *dev; - int wake = 0; TRACE_ENTRY(); @@ -2942,29 +3052,39 @@ static void scst_unblock_aborted_cmds(int scst_mutex_held) list_for_each_entry(dev, &scst_dev_list, dev_list_entry) { struct scst_cmd *cmd, *tcmd; + struct scst_tgt_dev *tgt_dev; spin_lock_bh(&dev->dev_lock); + local_irq_disable(); list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list, blocked_cmd_list_entry) { - if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) { + if (__scst_check_unblock_aborted_cmd(cmd)) list_del(&cmd->blocked_cmd_list_entry); - TRACE_MGMT_DBG("Moving aborted blocked cmd %p " - "to active cmd list", cmd); - spin_lock_irq(&scst_list_lock); - list_move_tail(&cmd->cmd_list_entry, - &scst_active_cmd_list); - spin_unlock_irq(&scst_list_lock); - wake = 1; - } } + local_irq_enable(); spin_unlock_bh(&dev->dev_lock); + + local_irq_disable(); + list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, + dev_tgt_dev_list_entry) { + spin_lock_bh(&tgt_dev->sn_lock); + list_for_each_entry_safe(cmd, tcmd, + &tgt_dev->deferred_cmd_list, + sn_cmd_list_entry) { + if (__scst_check_unblock_aborted_cmd(cmd)) { + TRACE_MGMT_DBG("Deleting aborted SN " + "cmd %p from SN list", cmd); + tgt_dev->def_cmd_count--; + list_del(&cmd->sn_cmd_list_entry); + } + } + spin_unlock_bh(&tgt_dev->sn_lock); + } + local_irq_enable(); } if (!scst_mutex_held) up(&scst_mutex); - if (wake) - wake_up(&scst_list_waitQ); - TRACE_EXIT(); return; } @@ -2978,21 +3098,17 @@ static void __scst_abort_task_set(struct scst_mgmt_cmd *mcmd, TRACE_ENTRY(); - local_bh_disable(); - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&sess->sess_list_lock); TRACE_DBG("Searching in search cmd list (sess=%p)", sess); list_for_each_entry(cmd, &sess->search_cmd_list, search_cmd_list_entry) { - if ((cmd->tgt_dev == NULL) && - (cmd->lun == tgt_dev->acg_dev->lun)) - continue; - if (cmd->tgt_dev != tgt_dev) - continue; - scst_abort_cmd(cmd, mcmd, other_ini, 0); + if ((cmd->tgt_dev == tgt_dev) || + ((cmd->tgt_dev == NULL) && + (cmd->lun == tgt_dev->acg_dev->lun))) + scst_abort_cmd(cmd, mcmd, other_ini, 0); } - spin_unlock_irq(&scst_list_lock); - local_bh_enable(); + spin_unlock_irq(&sess->sess_list_lock); scst_unblock_aborted_cmds(scst_mutex_held); @@ -3023,23 +3139,18 @@ static int scst_abort_task_set(struct scst_mgmt_cmd *mcmd) return res; } -static int scst_check_delay_mgmt_cmd(struct scst_mgmt_cmd *mcmd, int locked) +static int scst_check_delay_mgmt_cmd(struct scst_mgmt_cmd *mcmd) { - /* - * No need for special protection for SCST_FLAG_TM_ACTIVE, since - * we could be called from the only thread. - */ - if (test_bit(SCST_FLAG_TM_ACTIVE, &scst_flags)) { - TRACE_MGMT_DBG("Moving mgmt cmd %p to delayed mgmt cmd list", + if (test_bit(SCST_FLAG_TM_ACTIVE, &scst_flags) && !mcmd->active) { + TRACE_MGMT_DBG("Adding mgmt cmd %p to delayed mgmt cmd list", mcmd); - if (!locked) - spin_lock_irq(&scst_list_lock); - list_move_tail(&mcmd->mgmt_cmd_list_entry, + spin_lock_irq(&scst_mcmd_lock); + list_add_tail(&mcmd->mgmt_cmd_list_entry, &scst_delayed_mgmt_cmd_list); - if (!locked) - spin_unlock_irq(&scst_list_lock); + spin_unlock_irq(&scst_mcmd_lock); return -1; } else { + mcmd->active = 1; set_bit(SCST_FLAG_TM_ACTIVE, &scst_flags); return 0; } @@ -3053,7 +3164,7 @@ static int scst_mgmt_cmd_init(struct scst_mgmt_cmd *mcmd) TRACE_ENTRY(); - res = scst_check_delay_mgmt_cmd(mcmd, 1); + res = scst_check_delay_mgmt_cmd(mcmd); if (res != 0) goto out; @@ -3061,24 +3172,26 @@ static int scst_mgmt_cmd_init(struct scst_mgmt_cmd *mcmd) struct scst_session *sess = mcmd->sess; struct scst_cmd *cmd; - local_bh_disable(); - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&sess->sess_list_lock); cmd = __scst_find_cmd_by_tag(sess, mcmd->tag); if (cmd == NULL) { TRACE(TRACE_MGMT, "ABORT TASK failed: command for " "tag %d not found", mcmd->tag); mcmd->status = SCST_MGMT_STATUS_TASK_NOT_EXIST; mcmd->state = SCST_MGMT_CMD_STATE_DONE; - } else { - TRACE(TRACE_MGMT, "Cmd %p for tag %d (sn %d) found, " - "aborting it", cmd, mcmd->tag, cmd->sn); - mcmd->cmd_to_abort = cmd; - scst_abort_cmd(cmd, mcmd, 0, 1); - res = scst_set_mcmd_next_state(mcmd); - mcmd->cmd_to_abort = NULL; /* just in case */ + spin_unlock_irq(&sess->sess_list_lock); + goto out; } - spin_unlock_irq(&scst_list_lock); - local_bh_enable(); + scst_cmd_get(cmd); + spin_unlock_irq(&sess->sess_list_lock); + TRACE(TRACE_MGMT, "Cmd %p for tag %d (sn %d) found, " + "aborting it", cmd, mcmd->tag, cmd->sn); + mcmd->cmd_to_abort = cmd; + scst_abort_cmd(cmd, mcmd, 0, 1); + scst_unblock_aborted_cmds(0); + res = scst_set_mcmd_next_state(mcmd); + mcmd->cmd_to_abort = NULL; /* just in case */ + scst_cmd_put(cmd); } else { int rc; rc = scst_mgmt_translate_lun(mcmd); @@ -3180,10 +3293,8 @@ static int scst_target_reset(struct scst_mgmt_cmd *mcmd) up(&scst_mutex); - spin_lock_irq(&scst_list_lock); - tm_dbg_task_mgmt("TARGET RESET"); + tm_dbg_task_mgmt("TARGET RESET", 0); res = scst_set_mcmd_next_state(mcmd); - spin_unlock_irq(&scst_list_lock); TRACE_EXIT_RES(res); return res; @@ -3220,10 +3331,8 @@ static int scst_lun_reset(struct scst_mgmt_cmd *mcmd) } out_tm_dbg: - spin_lock_irq(&scst_list_lock); - tm_dbg_task_mgmt("LUN RESET"); + tm_dbg_task_mgmt("LUN RESET", 0); res = scst_set_mcmd_next_state(mcmd); - spin_unlock_irq(&scst_list_lock); TRACE_EXIT_RES(res); return res; @@ -3268,9 +3377,7 @@ static int scst_abort_all_nexus_loss_sess(struct scst_mgmt_cmd *mcmd, } up(&scst_mutex); - spin_lock_irq(&scst_list_lock); res = scst_set_mcmd_next_state(mcmd); - spin_unlock_irq(&scst_list_lock); TRACE_EXIT_RES(res); return res; @@ -3323,9 +3430,7 @@ static int scst_abort_all_nexus_loss_tgt(struct scst_mgmt_cmd *mcmd, up(&scst_mutex); - spin_lock_irq(&scst_list_lock); res = scst_set_mcmd_next_state(mcmd); - spin_unlock_irq(&scst_list_lock); TRACE_EXIT_RES(res); return res; @@ -3376,16 +3481,21 @@ static int scst_mgmt_cmd_exec(struct scst_mgmt_cmd *mcmd) mcmd->status = SCST_MGMT_STATUS_FN_NOT_SUPPORTED; /* Nothing to do (yet) */ } - break; + goto out_done; default: PRINT_ERROR_PR("Unknown task management function %d", mcmd->fn); mcmd->status = SCST_MGMT_STATUS_REJECTED; - break; + goto out_done; } +out: TRACE_EXIT_RES(res); return res; + +out_done: + mcmd->state = SCST_MGMT_CMD_STATE_DONE; + goto out; } static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd) @@ -3396,16 +3506,16 @@ static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd) TRACE_ENTRY(); clear_bit(SCST_FLAG_TM_ACTIVE, &scst_flags); + spin_lock_irq(&scst_mcmd_lock); if (!list_empty(&scst_delayed_mgmt_cmd_list)) { struct scst_mgmt_cmd *m; - spin_lock_irq(&scst_list_lock); m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m), mgmt_cmd_list_entry); - TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to active mgmt " - "cmd list", m); + TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active " + "mgmt cmd list", m); list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list); - spin_unlock_irq(&scst_list_lock); } + spin_unlock_irq(&scst_mcmd_lock); mcmd->state = SCST_MGMT_CMD_STATE_FINISHED; if (scst_is_strict_mgmt_fn(mcmd->fn) && (mcmd->completed_cmd_count > 0)) @@ -3483,48 +3593,41 @@ static int scst_process_mgmt_cmd(struct scst_mgmt_cmd *mcmd) scst_mgmt_cmd_send_done(mcmd); break; + default: + PRINT_ERROR_PR("Unknown state %d of management command", + mcmd->state); + res = -1; + /* go through */ case SCST_MGMT_CMD_STATE_FINISHED: - goto out_free; + scst_free_mgmt_cmd(mcmd); + goto out; #ifdef EXTRACHECKS case SCST_MGMT_CMD_STATE_EXECUTING: sBUG(); #endif - - default: - PRINT_ERROR_PR("Unknown state %d of management command", - mcmd->state); - res = -1; - goto out_free; } } out: TRACE_EXIT_RES(res); return res; - -out_free: - scst_free_mgmt_cmd(mcmd, 1); - goto out; } static inline int test_mgmt_cmd_list(void) { - int res = (!list_empty(&scst_active_mgmt_cmd_list) && - !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) || + int res = !list_empty(&scst_active_mgmt_cmd_list) || unlikely(kthread_should_stop()); return res; } int scst_mgmt_cmd_thread(void *arg) { - struct scst_mgmt_cmd *mcmd; - TRACE_ENTRY(); current->flags |= PF_NOFREEZE; - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&scst_mcmd_lock); while(!kthread_should_stop()) { wait_queue_t wait; init_waitqueue_entry(&wait, current); @@ -3536,36 +3639,42 @@ int scst_mgmt_cmd_thread(void *arg) set_current_state(TASK_INTERRUPTIBLE); if (test_mgmt_cmd_list()) break; - spin_unlock_irq(&scst_list_lock); + spin_unlock_irq(&scst_mcmd_lock); schedule(); - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&scst_mcmd_lock); } set_current_state(TASK_RUNNING); remove_wait_queue(&scst_mgmt_cmd_list_waitQ, &wait); } - while (!list_empty(&scst_active_mgmt_cmd_list) && - !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) - { + while (!list_empty(&scst_active_mgmt_cmd_list)) { int rc; + struct scst_mgmt_cmd *mcmd; mcmd = list_entry(scst_active_mgmt_cmd_list.next, typeof(*mcmd), mgmt_cmd_list_entry); - TRACE_MGMT_DBG("Moving mgmt cmd %p to mgmt cmd list", - mcmd); - list_move_tail(&mcmd->mgmt_cmd_list_entry, - &scst_mgmt_cmd_list); - spin_unlock_irq(&scst_list_lock); + TRACE_MGMT_DBG("Deleting mgmt cmd %p from active cmd " + "list", mcmd); + list_del(&mcmd->mgmt_cmd_list_entry); + spin_unlock_irq(&scst_mcmd_lock); rc = scst_process_mgmt_cmd(mcmd); - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&scst_mcmd_lock); if (rc > 0) { - TRACE_MGMT_DBG("Moving mgmt cmd %p to head " - "of active mgmt cmd list", mcmd); - list_move(&mcmd->mgmt_cmd_list_entry, - &scst_active_mgmt_cmd_list); + if (test_bit(SCST_FLAG_SUSPENDED, &scst_flags) && + !test_bit(SCST_FLAG_SUSPENDING, &scst_flags)) { + TRACE_MGMT_DBG("Adding mgmt cmd %p to head " + "of delayed mgmt cmd list", mcmd); + list_add(&mcmd->mgmt_cmd_list_entry, + &scst_delayed_mgmt_cmd_list); + } else { + TRACE_MGMT_DBG("Adding mgmt cmd %p to head " + "of active mgmt cmd list", mcmd); + list_add(&mcmd->mgmt_cmd_list_entry, + &scst_active_mgmt_cmd_list); + } } } } - spin_unlock_irq(&scst_list_lock); + spin_unlock_irq(&scst_mcmd_lock); /* * If kthread_should_stop() is true, we are guaranteed to be @@ -3614,8 +3723,9 @@ static int scst_post_rx_mgmt_cmd(struct scst_session *sess, scst_sess_get(sess); - spin_lock_irqsave(&scst_list_lock, flags); + local_irq_save(flags); + spin_lock(&sess->sess_list_lock); sess->sess_cmd_count++; #ifdef EXTRACHECKS @@ -3644,10 +3754,14 @@ static int scst_post_rx_mgmt_cmd(struct scst_session *sess, } } - TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd list", mcmd); - list_add_tail(&mcmd->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list); + spin_unlock(&sess->sess_list_lock); - spin_unlock_irqrestore(&scst_list_lock, flags); + TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd list", mcmd); + spin_lock(&scst_mcmd_lock); + list_add_tail(&mcmd->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list); + spin_unlock(&scst_mcmd_lock); + + local_irq_restore(flags); wake_up(&scst_mgmt_cmd_list_waitQ); @@ -3656,7 +3770,8 @@ out: return res; out_unlock: - spin_unlock_irqrestore(&scst_list_lock, flags); + spin_unlock(&sess->sess_list_lock); + local_irq_restore(flags); goto out; } @@ -3699,7 +3814,7 @@ out: return res; out_free: - scst_free_mgmt_cmd(mcmd, 0); + scst_free_mgmt_cmd(mcmd); mcmd = NULL; goto out; } @@ -3740,7 +3855,7 @@ out: return res; out_free: - scst_free_mgmt_cmd(mcmd, 0); + scst_free_mgmt_cmd(mcmd); mcmd = NULL; goto out; } @@ -3780,7 +3895,8 @@ static int scst_init_session(struct scst_session *sess) int mwake = 0; TRACE_ENTRY(); - + + scst_suspend_activity(); down(&scst_mutex); if (sess->initiator_name) { @@ -3804,14 +3920,26 @@ static int scst_init_session(struct scst_session *sess) res = scst_sess_alloc_tgt_devs(sess); up(&scst_mutex); - + scst_resume_activity(); + if (sess->init_result_fn) { TRACE_DBG("Calling init_result_fn(%p)", sess); sess->init_result_fn(sess, sess->reg_sess_data, res); TRACE_DBG("%s", "init_result_fn() returned"); } - spin_lock_irq(&scst_list_lock); +#ifdef CONFIG_LOCKDEP + if (res == 0) { + sess->shutdown_compl = kmalloc(sizeof(*sess->shutdown_compl), + GFP_KERNEL); + if (sess->shutdown_compl == NULL) + res = -ENOMEM; + else + init_completion(sess->shutdown_compl); + } +#endif + + spin_lock_irq(&sess->sess_list_lock); if (res == 0) sess->init_phase = SCST_SESS_IPH_SUCCESS; @@ -3826,12 +3954,13 @@ restart: list_del(&cmd->cmd_list_entry); sess->sess_cmd_count--; list_del(&cmd->search_cmd_list_entry); - spin_unlock_irq(&scst_list_lock); + spin_unlock_irq(&sess->sess_list_lock); scst_cmd_init_done(cmd, SCST_CONTEXT_THREAD); - spin_lock_irq(&scst_list_lock); + spin_lock_irq(&sess->sess_list_lock); goto restart; } + spin_lock(&scst_mcmd_lock); list_for_each_entry_safe(mcmd, tm, &sess->init_deferred_mcmd_list, mgmt_cmd_list_entry) { @@ -3841,8 +3970,9 @@ restart: &scst_active_mgmt_cmd_list); mwake = 1; } + spin_unlock(&scst_mcmd_lock); sess->init_phase = SCST_SESS_IPH_READY; - spin_unlock_irq(&scst_list_lock); + spin_unlock_irq(&sess->sess_list_lock); if (mwake) wake_up(&scst_mgmt_cmd_list_waitQ); @@ -3904,28 +4034,47 @@ void scst_unregister_session(struct scst_session *sess, int wait, void (*unreg_done_fn) (struct scst_session *sess)) { unsigned long flags; + struct completion *pc; +#ifndef CONFIG_LOCKDEP DECLARE_COMPLETION(c); +#endif TRACE_ENTRY(); +#ifdef CONFIG_LOCKDEP + pc = sess->shutdown_compl; +#else + pc = &c; +#endif + spin_lock_irqsave(&scst_mgmt_lock, flags); sess->shutting_down = 1; sess->unreg_done_fn = unreg_done_fn; if (wait) { - sess->shutdown_compl = &c; + sess->shutdown_compl = pc; smp_mb(); } +#ifdef CONFIG_LOCKDEP + else + sess->shutdown_compl = NULL; +#endif spin_unlock_irqrestore(&scst_mgmt_lock, flags); + tm_dbg_task_mgmt("UNREGISTER SESSION", 1); + scst_sess_put(sess); if (wait) { TRACE_DBG("Waiting for session %p to complete", sess); - wait_for_completion(&c); + wait_for_completion(pc); } +#ifdef CONFIG_LOCKDEP + kfree(pc); +#endif + TRACE_EXIT(); return; } @@ -3963,7 +4112,6 @@ int scst_mgmt_thread(void *arg) set_current_state(TASK_RUNNING); remove_wait_queue(&scst_mgmt_waitQ, &wait); } - restart: list_for_each_entry(sess, &scst_sess_mgmt_list, sess_mgmt_list_entry) @@ -3999,7 +4147,7 @@ restart: return 0; } -/* Called under scst_list_lock */ +/* Called under sess->sess_list_lock */ struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess, uint32_t tag) { struct scst_cmd *cmd = NULL; @@ -4033,7 +4181,7 @@ struct scst_cmd *scst_find_cmd(struct scst_session *sess, void *data, if (cmp_fn == NULL) goto out; - spin_lock_irqsave(&scst_list_lock, flags); + spin_lock_irqsave(&sess->sess_list_lock, flags); TRACE_DBG("Searching in search cmd list (sess=%p)", sess); list_for_each_entry(cmd, &sess->search_cmd_list, @@ -4045,7 +4193,7 @@ struct scst_cmd *scst_find_cmd(struct scst_session *sess, void *data, cmd = NULL; out_unlock: - spin_unlock_irqrestore(&scst_list_lock, flags); + spin_unlock_irqrestore(&sess->sess_list_lock, flags); out: TRACE_EXIT(); @@ -4057,9 +4205,9 @@ struct scst_cmd *scst_find_cmd_by_tag(struct scst_session *sess, { unsigned long flags; struct scst_cmd *cmd; - spin_lock_irqsave(&scst_list_lock, flags); + spin_lock_irqsave(&sess->sess_list_lock, flags); cmd = __scst_find_cmd_by_tag(sess, tag); - spin_unlock_irqrestore(&scst_list_lock, flags); + spin_unlock_irqrestore(&sess->sess_list_lock, flags); return cmd; } @@ -4067,16 +4215,16 @@ void *scst_cmd_get_tgt_priv_lock(struct scst_cmd *cmd) { void *res; unsigned long flags; - spin_lock_irqsave(&scst_list_lock, flags); + spin_lock_irqsave(&scst_main_lock, flags); res = cmd->tgt_priv; - spin_unlock_irqrestore(&scst_list_lock, flags); + spin_unlock_irqrestore(&scst_main_lock, flags); return res; } void scst_cmd_set_tgt_priv_lock(struct scst_cmd *cmd, void *val) { unsigned long flags; - spin_lock_irqsave(&scst_list_lock, flags); + spin_lock_irqsave(&scst_main_lock, flags); cmd->tgt_priv = val; - spin_unlock_irqrestore(&scst_list_lock, flags); + spin_unlock_irqrestore(&scst_main_lock, flags); }