From b3aaaed00d0a4d4cb23a492671c560423b86f1ec Mon Sep 17 00:00:00 2001 From: Vladislav Bolkhovitin Date: Wed, 21 Feb 2007 12:50:48 +0000 Subject: [PATCH] Execution context cleanup. Completed full support for SCSI task attributes (SIMPLE, ORDERED, etc.) + minor cleanups git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@91 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- scst/ChangeLog | 6 + scst/include/scsi_tgt.h | 230 +++++---- scst/src/Makefile | 1 + scst/src/dev_handlers/scst_fileio.c | 723 ++++++++++------------------ scst/src/scst.c | 11 +- scst/src/scst_lib.c | 457 ++++++++++++++---- scst/src/scst_priv.h | 84 ++-- scst/src/scst_proc.c | 8 + scst/src/scst_targ.c | 358 +++++++++----- 9 files changed, 1090 insertions(+), 788 deletions(-) diff --git a/scst/ChangeLog b/scst/ChangeLog index ca193267c..f1779963e 100644 --- a/scst/ChangeLog +++ b/scst/ChangeLog @@ -1,6 +1,12 @@ Summary of changes between versions 0.9.5 and 0.9.6 --------------------------------------------------- + - Internal locking and execution context were reimplemnted. Particularly, + implemented full support for SCSI task attributes (SIMPLE, ORDERED, + etc.). + + - Updated to work on 2.6.20.x + - Updated to work on 2.6.19.x, thanks to Ming Zhang. - Internal threads management reimplemented based on kthread*() API, diff --git a/scst/include/scsi_tgt.h b/scst/include/scsi_tgt.h index d4922d658..31b310667 100644 --- a/scst/include/scsi_tgt.h +++ b/scst/include/scsi_tgt.h @@ -49,12 +49,6 @@ /* LUN translation (cmd->tgt_dev assignment) */ #define SCST_CMD_STATE_INIT 2 -/* - * Again LUN translation (cmd->tgt_dev assignment), used if dev handler - * wants to restart cmd on another LUN - */ -#define SCST_CMD_STATE_REINIT 3 - /* Dev handler's parse() is going to be called */ #define SCST_CMD_STATE_DEV_PARSE 4 @@ -380,6 +374,8 @@ #define SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC 9 #define SCST_TGT_DEV_AFTER_EXEC_ATOMIC 10 +/* Set HEAD OF QUEUE cmd is being executed */ +#define SCST_TGT_DEV_HQ_ACTIVE 12 #ifdef DEBUG_TM #define SCST_TGT_DEV_UNDER_TM_DBG 20 @@ -734,12 +730,6 @@ struct scst_tgt_template struct scst_dev_type { - /* Name of the dev handler. Must be unique. MUST HAVE */ - char name[15]; - - /* SCSI type of the supported device. MUST HAVE */ - int type; - /* * True, if corresponding function supports execution in * the atomic (non-sleeping) context @@ -747,24 +737,7 @@ struct scst_dev_type unsigned parse_atomic:1; unsigned exec_atomic:1; unsigned dev_done_atomic:1; - - /* - * Called when new device is attaching to the dev handler - * Returns 0 on success, error code otherwise. - */ - int (*attach) (struct scst_device *dev); - - /* Called when new device is detaching from the dev handler */ - void (*detach) (struct scst_device *dev); - - /* - * Called when new tgt_dev (session) is attaching to the dev handler. - * Returns 0 on success, error code otherwise. - */ - int (*attach_tgt) (struct scst_tgt_dev *tgt_dev); - - /* Called when tgt_dev (session) is detaching from the dev handler */ - void (*detach_tgt) (struct scst_tgt_dev *tgt_dev); + unsigned dedicated_thread:1; /* * Called to parse CDB from the cmd and initialize @@ -841,6 +814,24 @@ struct scst_dev_type int (*task_mgmt_fn) (struct scst_mgmt_cmd *mgmt_cmd, struct scst_tgt_dev *tgt_dev); + /* + * Called when new device is attaching to the dev handler + * Returns 0 on success, error code otherwise. + */ + int (*attach) (struct scst_device *dev); + + /* Called when new device is detaching from the dev handler */ + void (*detach) (struct scst_device *dev); + + /* + * Called when new tgt_dev (session) is attaching to the dev handler. + * Returns 0 on success, error code otherwise. + */ + int (*attach_tgt) (struct scst_tgt_dev *tgt_dev); + + /* Called when tgt_dev (session) is detaching from the dev handler */ + void (*detach_tgt) (struct scst_tgt_dev *tgt_dev); + /* * Those functions can be used to export the handler's statistics and * other infos to the world outside the kernel as well as to get some @@ -852,6 +843,12 @@ struct scst_dev_type int (*write_proc) (char *buffer, char **start, off_t offset, int length, int *eof, struct scst_dev_type *dev_type); + /* Name of the dev handler. Must be unique. MUST HAVE */ + char name[15]; + + /* SCSI type of the supported device. MUST HAVE */ + int type; + struct module *module; /* private: */ @@ -1094,6 +1091,12 @@ struct scst_cmd /* Set if the cmd was done or aborted out of its SN */ unsigned long out_of_sn:1; + /* Set if the cmd is HEAD OF QUEUE */ + unsigned long head_of_queue:1; + + /* Set if the cmd is deferred HEAD OF QUEUE */ + unsigned long hq_deferred:1; + /**************************************************************/ unsigned long cmd_flags; /* cmd's async flags */ @@ -1116,6 +1119,9 @@ struct scst_cmd /* Cmd's serial number, used to execute cmd's in order of arrival */ unsigned int sn; + /* The corresponding sn_slot in tgt_dev->sn_slots */ + atomic_t *sn_slot; + /* List entry for session's search_cmd_list */ struct list_head search_cmd_list_entry; @@ -1193,18 +1199,6 @@ struct scst_cmd /* Used for storage of dev handler private stuff */ void *dh_priv; - /* - * Fileio private fields - */ - struct list_head fileio_cmd_list_entry; - int fileio_in_list; - - /* - * Used to store previous tgt_dev if dev handler returns - * SCST_CMD_STATE_REINIT state - */ - struct scst_tgt_dev *tgt_dev_saved; - struct scst_cmd *orig_cmd; /* Used to issue REQUEST SENSE */ }; @@ -1253,8 +1247,7 @@ struct scst_device { struct scst_dev_type *handler; /* corresponding dev handler */ - /* Used to translate SCSI's cmd to SCST's cmd */ - struct gendisk *rq_disk; + unsigned short type; /* SCSI type of the device */ /************************************************************* ** Dev's flags. Updates serialized by dev_lock or suspended @@ -1262,16 +1255,16 @@ struct scst_device *************************************************************/ /* Set if dev is RESERVED */ - unsigned int dev_reserved:1; + unsigned short dev_reserved:1; /* Set if dev accepts only one command at time */ - unsigned int dev_serialized:1; + unsigned short dev_serialized:1; /* Set if double reset UA is possible */ - unsigned int dev_double_ua_possible:1; + unsigned short dev_double_ua_possible:1; /* Set if reset UA sent (to avoid double reset UAs) */ - unsigned int dev_reset_ua_sent:1; + unsigned short dev_reset_ua_sent:1; /**************************************************************/ @@ -1290,13 +1283,16 @@ struct scst_device atomic_t on_dev_count; struct list_head blocked_cmd_list; /* protected by dev_lock */ - - /* Corresponding real SCSI device, could be NULL for virtual devices */ - struct scsi_device *scsi_dev; - + /* Used for storage of dev handler private stuff */ void *dh_priv; + /* Used to translate SCSI's cmd to SCST's cmd */ + struct gendisk *rq_disk; + + /* Corresponding real SCSI device, could be NULL for virtual devices */ + struct scsi_device *scsi_dev; + /* Used to wait for requested amount of "on_dev" commands */ wait_queue_head_t on_dev_waitQ; @@ -1322,6 +1318,19 @@ struct scst_device struct list_head dev_acg_dev_list; }; +/* + * Used to store threads local tgt_dev specific data + */ +struct scst_thr_data_hdr +{ + /* List entry in tgt_dev->thr_data_list */ + struct list_head thr_data_list_entry; + pid_t pid; /* PID of the owner thread */ + atomic_t ref; + /* Function that will be called on the tgt_dev destruction */ + void (*free_fn) (struct scst_thr_data_hdr *data); +}; + /* * Used to store per-session specific device information */ @@ -1329,48 +1338,64 @@ struct scst_tgt_dev { /* List entry in sess->sess_tgt_dev_list */ struct list_head sess_tgt_dev_list_entry; - - struct scst_acg_dev *acg_dev; /* corresponding acg_dev */ + + struct scst_device *dev; /* to save extra dereferences */ + lun_t lun; /* to save extra dereferences */ + + /* Pointer to lists of commands with the lock */ + struct scst_cmd_lists *p_cmd_lists; /* How many cmds alive on this dev in this session */ atomic_t cmd_count; + unsigned long tgt_dev_flags; /* tgt_dev's async flags */ + + /* + * Used to execute cmd's in order of arrival, honoring SCSI task + * attributes. + * + * Protected by sn_lock, except expected_sn, which is protected by + * itself. Curr_sn must have the same size as expected_sn to + * overflow simultaneously. + */ + int def_cmd_count; + spinlock_t sn_lock; + int expected_sn; + int curr_sn; + struct list_head deferred_cmd_list; + struct list_head skipped_sn_list; + struct list_head hq_cmd_list; + unsigned short prev_cmd_ordered; /* Set if the prev cmd was ORDERED */ + short num_free_sn_slots; + atomic_t *cur_sn_slot; + atomic_t sn_slots[10]; + + /* Lists of commands with the lock, if dedicated threads are used */ + struct scst_cmd_lists cmd_lists; + + /* Used for storage of dev handler private stuff */ + void *dh_priv; + + /* List of scst_thr_data_hdr and lock */ + spinlock_t thr_data_lock; + struct list_head thr_data_list; + spinlock_t tgt_dev_lock; /* per-session device lock */ /* List of UA's for this device, protected by tgt_dev_lock */ struct list_head UA_list; - unsigned long tgt_dev_flags; /* tgt_dev's flags */ - - /* Used for storage of dev handler private stuff */ - void *dh_priv; - - /* Pointer to lists of commands with the lock */ - struct scst_cmd_lists *p_cmd_lists; - - /* - * Used to execute cmd's in order of arrival. - * - * Protected by sn_lock, except curr_sn and expected_sn. - * Expected_sn protected by itself, since only one thread, which - * processes SN matching command, can increment it at any time. - * Next_sn must have the same size as expected_sn to overflow - * simultaneously. - */ - atomic_t curr_sn; - int expected_sn; - spinlock_t sn_lock; - int def_cmd_count; - struct list_head deferred_cmd_list; - struct list_head skipped_sn_list; - struct scst_session *sess; /* corresponding session */ - + struct scst_acg_dev *acg_dev; /* corresponding acg_dev */ + /* list entry in dev->dev_tgt_dev_list */ struct list_head dev_tgt_dev_list_entry; /* internal tmp list entry */ struct list_head extra_tgt_dev_list_entry; + + /* Dedicated thread. Doesn't need any protection. */ + struct task_struct *thread; }; /* @@ -1609,7 +1634,18 @@ struct scst_cmd *scst_rx_cmd(struct scst_session *sess, * Notifies SCST that the driver finished its part of the command * initialization, and the command is ready for execution. * The second argument sets preferred command execition context. - * See SCST_CONTEXT_* constants for details + * See SCST_CONTEXT_* constants for details. + * + * !!IMPORTANT!! + * + * If cmd->no_sn not set, this function, as well as scst_cmd_init_stage1_done() + * and scst_restart_cmd() must not be called simultaneously for the same session + * (more precisely, for the same session/LUN, i.e. tgt_dev), i.e. they must be + * somehow externally serialized. This is needed to have lock free fast path in + * scst_cmd_set_sn(). For majority of targets those functions are naturally + * serialized by the single source of commands. Only iSCSI immediate commands + * with multiple connections per session seems to be an exception. For it, some + * mutex/lock shall be used for the serialization. */ void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context); @@ -1619,6 +1655,8 @@ void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context); * SCST done the command's preprocessing preprocessing_done() function * should be called. The second argument sets preferred command execition * context. See SCST_CONTEXT_* constants for details. + * + * See also scst_cmd_init_done() comment for the serialization requirements. */ static inline void scst_cmd_init_stage1_done(struct scst_cmd *cmd, int pref_context, int set_sn) @@ -1634,7 +1672,9 @@ static inline void scst_cmd_init_stage1_done(struct scst_cmd *cmd, * The second argument sets data receiving completion status * (see SCST_PREPROCESS_STATUS_* constants for details) * The third argument sets preferred command execition context - * (see SCST_CONTEXT_* constants for details) + * (see SCST_CONTEXT_* constants for details). + * + * See also scst_cmd_init_done() comment for the serialization requirements. */ void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context); @@ -2231,4 +2271,32 @@ struct scatterlist *scst_alloc(int size, unsigned long gfp_mask, /* Frees SG vector returned by scst_alloc() */ void scst_free(struct scatterlist *sg, int count); +/* + * Adds local to the current thread data to tgt_dev + * (they will be local for the tgt_dev and current thread). + */ +void scst_add_thr_data(struct scst_tgt_dev *tgt_dev, + struct scst_thr_data_hdr *data, + void (*free_fn) (struct scst_thr_data_hdr *data)); + +/* Deletes all local to threads data from tgt_dev */ +void scst_del_all_thr_data(struct scst_tgt_dev *tgt_dev); + +/* Deletes all local to threads data from all tgt_dev's of the dev */ +void scst_dev_del_all_thr_data(struct scst_device *dev); + +/* Finds local to the current thread data. Returns NULL, if they not found. */ +struct scst_thr_data_hdr *scst_find_thr_data(struct scst_tgt_dev *tgt_dev); + +static inline void scst_thr_data_get(struct scst_thr_data_hdr *data) +{ + atomic_inc(&data->ref); +} + +static inline void scst_thr_data_put(struct scst_thr_data_hdr *data) +{ + if (atomic_dec_and_test(&data->ref)) + data->free_fn(data); +} + #endif /* __SCST_H */ diff --git a/scst/src/Makefile b/scst/src/Makefile index 4035112ad..c5434ce11 100644 --- a/scst/src/Makefile +++ b/scst/src/Makefile @@ -105,6 +105,7 @@ EXTRA_CFLAGS += -DDEBUG #EXTRA_CFLAGS += -DDEBUG_TM -DTM_DBG_GO_OFFLINE=0 #EXTRA_CFLAGS += -DDEBUG_RETRY #EXTRA_CFLAGS += -DDEBUG_OOM +#EXTRA_CFLAGS += -DDEBUG_SN # If defined, makes SCST zero allocated data buffers. # Undefining it considerably improves performance and eases CPU load, diff --git a/scst/src/dev_handlers/scst_fileio.c b/scst/src/dev_handlers/scst_fileio.c index 251cc1985..3b5f60359 100644 --- a/scst/src/dev_handlers/scst_fileio.c +++ b/scst/src/dev_handlers/scst_fileio.c @@ -51,6 +51,12 @@ static struct scst_proc_log fileio_proc_local_trace_tbl[] = #include "scst_dev_handler.h" +#if defined(DEBUG) && defined(CONFIG_DEBUG_SLAB) +#define FILEIO_SLAB_FLAGS ( SLAB_RED_ZONE | SLAB_POISON ) +#else +#define FILEIO_SLAB_FLAGS 0L +#endif + /* 8 byte ASCII Vendor of the FILE IO target */ #define SCST_FIO_VENDOR "SCST_FIO" /* 4 byte ASCII Product Revision Level of the FILE IO target - left aligned */ @@ -94,6 +100,11 @@ struct scst_fileio_dev { uint64_t nblocks; int block_shift; loff_t file_size; /* in bytes */ + spinlock_t flags_lock; + /* + * Below flags are protected by flags_lock or suspended activity + * with scst_fileio_mutex. + */ unsigned int rd_only_flag:1; unsigned int wt_flag:1; unsigned int nv_cache:1; @@ -107,37 +118,38 @@ struct scst_fileio_dev { must be <= SCSI Model + 1 */ char *file_name; /* File name */ char *usn; + struct scst_device *dev; struct list_head fileio_dev_list_entry; - struct list_head ftgt_list; - struct semaphore ftgt_list_mutex; }; struct scst_fileio_tgt_dev { - spinlock_t fdev_lock; enum scst_cmd_queue_type last_write_cmd_queue_type; +}; + +struct scst_fileio_thr { + struct scst_thr_data_hdr hdr; struct file *fd; struct iovec *iv; int iv_count; - struct list_head fdev_cmd_list; - wait_queue_head_t fdev_waitQ; - struct task_struct *cmd_thread; struct scst_fileio_dev *virt_dev; - struct list_head ftgt_list_entry; }; +static struct kmem_cache *fileio_thr_cachep; + static int fileio_attach(struct scst_device *dev); static void fileio_detach(struct scst_device *dev); static int fileio_attach_tgt(struct scst_tgt_dev *tgt_dev); static void fileio_detach_tgt(struct scst_tgt_dev *tgt_dev); static int disk_fileio_parse(struct scst_cmd *, const struct scst_info_cdb *info_cdb); -static int disk_fileio_exec(struct scst_cmd *cmd); +static int fileio_do_job(struct scst_cmd *cmd); static int cdrom_fileio_parse(struct scst_cmd *, const struct scst_info_cdb *info_cdb); static int cdrom_fileio_exec(struct scst_cmd *cmd); -static void fileio_exec_read(struct scst_cmd *cmd, loff_t loff); -static void fileio_exec_write(struct scst_cmd *cmd, loff_t loff); -static void fileio_exec_verify(struct scst_cmd *cmd, loff_t loff); -static int fileio_task_mgmt_fn(struct scst_mgmt_cmd *mcmd, - struct scst_tgt_dev *tgt_dev); +static void fileio_exec_read(struct scst_cmd *cmd, + struct scst_fileio_thr *thr, loff_t loff); +static void fileio_exec_write(struct scst_cmd *cmd, + struct scst_fileio_thr *thr, loff_t loff); +static void fileio_exec_verify(struct scst_cmd *cmd, + struct scst_fileio_thr *thr, loff_t loff); static void fileio_exec_read_capacity(struct scst_cmd *cmd); static void fileio_exec_read_capacity16(struct scst_cmd *cmd); static void fileio_exec_inquiry(struct scst_cmd *cmd); @@ -145,7 +157,7 @@ static void fileio_exec_mode_sense(struct scst_cmd *cmd); static void fileio_exec_mode_select(struct scst_cmd *cmd); static void fileio_exec_read_toc(struct scst_cmd *cmd); static void fileio_exec_prevent_allow_medium_removal(struct scst_cmd *cmd); -static int fileio_fsync(struct scst_fileio_tgt_dev *ftgt_dev, +static int fileio_fsync(struct scst_fileio_thr *thr, loff_t loff, loff_t len, struct scst_cmd *cmd); static int disk_fileio_read_proc(struct seq_file *seq, struct scst_dev_type *dev_type); static int disk_fileio_write_proc(char *buffer, char **start, off_t offset, @@ -158,15 +170,15 @@ static int cdrom_fileio_write_proc(char *buffer, char **start, off_t offset, name: DISK_FILEIO_NAME, \ type: TYPE_DISK, \ parse_atomic: 1, \ - exec_atomic: 1, \ + exec_atomic: 0, \ dev_done_atomic: 1, \ + dedicated_thread: 1, \ attach: fileio_attach, \ detach: fileio_detach, \ attach_tgt: fileio_attach_tgt, \ detach_tgt: fileio_detach_tgt, \ parse: disk_fileio_parse, \ - exec: disk_fileio_exec, \ - task_mgmt_fn: fileio_task_mgmt_fn, \ + exec: fileio_do_job, \ read_proc: disk_fileio_read_proc, \ write_proc: disk_fileio_write_proc, \ } @@ -175,15 +187,15 @@ static int cdrom_fileio_write_proc(char *buffer, char **start, off_t offset, name: CDROM_FILEIO_NAME, \ type: TYPE_ROM, \ parse_atomic: 1, \ - exec_atomic: 1, \ + exec_atomic: 0, \ dev_done_atomic: 1, \ + dedicated_thread: 1, \ attach: fileio_attach, \ detach: fileio_detach, \ attach_tgt: fileio_attach_tgt, \ detach_tgt: fileio_detach_tgt, \ parse: cdrom_fileio_parse, \ exec: cdrom_fileio_exec, \ - task_mgmt_fn: fileio_task_mgmt_fn, \ read_proc: cdrom_fileio_read_proc, \ write_proc: cdrom_fileio_write_proc,\ } @@ -271,8 +283,7 @@ static int fileio_attach(struct scst_device *dev) * scst_fileio_mutex must be already taken before * scst_register_virtual_device() */ - list_for_each_entry(vv, fileio_dev_list, fileio_dev_list_entry) - { + list_for_each_entry(vv, fileio_dev_list, fileio_dev_list_entry) { if (strcmp(vv->name, dev->virt_name) == 0) { virt_dev = vv; break; @@ -284,7 +295,9 @@ static int fileio_attach(struct scst_device *dev) res = -EINVAL; goto out; } - + + virt_dev->dev = dev; + if (dev->handler->type == TYPE_ROM) virt_dev->rd_only_flag = 1; @@ -393,6 +406,109 @@ static void fileio_detach(struct scst_device *dev) return; } +static void fileio_free_thr_data(struct scst_thr_data_hdr *d) +{ + struct scst_fileio_thr *thr = container_of(d, struct scst_fileio_thr, + hdr); + + TRACE_ENTRY(); + + if (thr->fd) + filp_close(thr->fd, NULL); + + if (thr->iv != NULL) + kfree(thr->iv); + + kmem_cache_free(fileio_thr_cachep, thr); + + TRACE_EXIT(); + return; +} + +static struct scst_fileio_thr *fileio_init_thr_data( + struct scst_tgt_dev *tgt_dev) +{ + struct scst_fileio_thr *res; + struct scst_fileio_dev *virt_dev = + (struct scst_fileio_dev *)tgt_dev->dev->dh_priv; + + TRACE_ENTRY(); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17) + res = kmem_cache_alloc(fileio_thr_cachep, GFP_KERNEL); + if (res != NULL) + memset(thr, 0, sizeof(*thr)); +#else + res = kmem_cache_zalloc(fileio_thr_cachep, GFP_KERNEL); +#endif + if (res == NULL) { + TRACE(TRACE_OUT_OF_MEM, "%s", "Unable to allocate struct " + "scst_fileio_thr"); + goto out; + } + + res->virt_dev = virt_dev; + + if (!virt_dev->cdrom_empty && !virt_dev->nullio) { + res->fd = fileio_open(virt_dev); + if (IS_ERR(res->fd)) { + PRINT_ERROR_PR("filp_open(%s) returned an error %ld", + virt_dev->file_name, PTR_ERR(res->fd)); + goto out_free; + } + } else + res->fd = NULL; + + scst_add_thr_data(tgt_dev, &res->hdr, fileio_free_thr_data); + +out: + TRACE_EXIT_HRES((unsigned long)res); + return res; + +out_free: + kmem_cache_free(fileio_thr_cachep, res); + res = NULL; + goto out; +} + +static int fileio_attach_tgt(struct scst_tgt_dev *tgt_dev) +{ + struct scst_fileio_tgt_dev *ftgt_dev; + int res = 0; + + TRACE_ENTRY(); + + ftgt_dev = kzalloc(sizeof(*ftgt_dev), GFP_KERNEL); + if (ftgt_dev == NULL) { + TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of per-session " + "virtual device failed"); + res = -ENOMEM; + goto out; + } + + tgt_dev->dh_priv = ftgt_dev; + +out: + TRACE_EXIT_RES(res); + return res; +} + +static void fileio_detach_tgt(struct scst_tgt_dev *tgt_dev) +{ + struct scst_fileio_tgt_dev *ftgt_dev = + (struct scst_fileio_tgt_dev *)tgt_dev->dh_priv; + + TRACE_ENTRY(); + + scst_del_all_thr_data(tgt_dev); + + kfree(ftgt_dev); + tgt_dev->dh_priv = NULL; + + TRACE_EXIT(); + return; +} + static inline int fileio_sync_queue_type(enum scst_cmd_queue_type qt) { switch(qt) { @@ -413,7 +529,7 @@ static inline int fileio_need_pre_sync(enum scst_cmd_queue_type cwqt, return 0; } -static void fileio_do_job(struct scst_cmd *cmd) +static int fileio_do_job(struct scst_cmd *cmd) { uint64_t lba_start = 0; loff_t data_len = 0; @@ -423,16 +539,33 @@ static void fileio_do_job(struct scst_cmd *cmd) struct scst_device *dev = cmd->dev; struct scst_fileio_dev *virt_dev = (struct scst_fileio_dev *)dev->dh_priv; + struct scst_thr_data_hdr *d; + struct scst_fileio_thr *thr = NULL; int fua = 0; TRACE_ENTRY(); + cmd->status = 0; + cmd->msg_status = 0; + cmd->host_status = DID_OK; + cmd->driver_status = 0; + if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) { TRACE_MGMT_DBG("Flag ABORTED set for " "cmd %p (tag %d), skipping", cmd, cmd->tag); goto done_uncompl; } - + + d = scst_find_thr_data(cmd->tgt_dev); + if (unlikely(d == NULL)) { + thr = fileio_init_thr_data(cmd->tgt_dev); + if (thr == NULL) { + scst_set_busy(cmd); + goto done; + } + scst_thr_data_get(&thr->hdr); + } else + thr = container_of(d, struct scst_fileio_thr, hdr); switch (opcode) { case READ_6: @@ -514,7 +647,7 @@ static void fileio_do_job(struct scst_cmd *cmd) case READ_10: case READ_12: case READ_16: - fileio_exec_read(cmd, loff); + fileio_exec_read(cmd, thr, loff); break; case WRITE_6: case WRITE_10: @@ -535,13 +668,13 @@ static void fileio_do_job(struct scst_cmd *cmd) cmd->queue_type, (uint64_t)loff, (uint64_t)data_len); do_fsync = 1; - if (fileio_fsync(ftgt_dev, 0, 0, cmd) != 0) + if (fileio_fsync(thr, 0, 0, cmd) != 0) goto done; } - fileio_exec_write(cmd, loff); + fileio_exec_write(cmd, thr, loff); /* O_SYNC flag is used for wt_flag devices */ if (do_fsync || fua) - fileio_fsync(ftgt_dev, loff, data_len, cmd); + fileio_fsync(thr, loff, data_len, cmd); } else { TRACE_DBG("%s", "Attempt to write to read-only device"); scst_set_cmd_error(cmd, @@ -566,15 +699,15 @@ static void fileio_do_job(struct scst_cmd *cmd) cmd->queue_type, (uint64_t)loff, (uint64_t)data_len); do_fsync = 1; - if (fileio_fsync(ftgt_dev, 0, 0, cmd) != 0) + if (fileio_fsync(thr, 0, 0, cmd) != 0) goto done; } - fileio_exec_write(cmd, loff); + fileio_exec_write(cmd, thr, loff); /* O_SYNC flag is used for wt_flag devices */ if (cmd->status == 0) - fileio_exec_verify(cmd, loff); + fileio_exec_verify(cmd, thr, loff); else if (do_fsync) - fileio_fsync(ftgt_dev, loff, data_len, cmd); + fileio_fsync(thr, loff, data_len, cmd); } else { TRACE_DBG("%s", "Attempt to write to read-only device"); scst_set_cmd_error(cmd, @@ -584,9 +717,6 @@ static void fileio_do_job(struct scst_cmd *cmd) case SYNCHRONIZE_CACHE: { int immed = cdb[1] & 0x2; - struct scst_fileio_tgt_dev *ftgt_dev = - (struct scst_fileio_tgt_dev*) - cmd->tgt_dev->dh_priv; TRACE(TRACE_ORDER, "SYNCHRONIZE_CACHE: " "loff=%Ld, data_len=%Ld, immed=%d", (uint64_t)loff, (uint64_t)data_len, immed); @@ -595,12 +725,12 @@ static void fileio_do_job(struct scst_cmd *cmd) cmd->completed = 1; cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT); /* cmd is dead here */ - fileio_fsync(ftgt_dev, loff, data_len, NULL); + fileio_fsync(thr, loff, data_len, NULL); /* ToDo: fileio_fsync() error processing */ scst_put(); goto out; } else { - fileio_fsync(ftgt_dev, loff, data_len, cmd); + fileio_fsync(thr, loff, data_len, cmd); break; } } @@ -608,7 +738,7 @@ static void fileio_do_job(struct scst_cmd *cmd) case VERIFY: case VERIFY_12: case VERIFY_16: - fileio_exec_verify(cmd, loff); + fileio_exec_verify(cmd, thr, loff); break; case MODE_SENSE: case MODE_SENSE_10: @@ -629,7 +759,21 @@ static void fileio_do_job(struct scst_cmd *cmd) case RESERVE_10: case RELEASE: case RELEASE_10: + case TEST_UNIT_READY: break; + case INQUIRY: + fileio_exec_inquiry(cmd); + break; + case READ_CAPACITY: + fileio_exec_read_capacity(cmd); + break; + case SERVICE_ACTION_IN: + if ((cmd->cdb[1] & 0x1f) == SAI_READ_CAPACITY_16) { + fileio_exec_read_capacity16(cmd); + break; + } + /* else go through */ + case REPORT_LUNS: default: TRACE_DBG("Invalid opcode %d", opcode); scst_set_cmd_error(cmd, @@ -643,158 +787,11 @@ done_uncompl: cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT); out: - TRACE_EXIT(); - return; -} - -static inline int test_cmd_list(struct scst_fileio_tgt_dev *ftgt_dev) -{ - int res = !list_empty(&ftgt_dev->fdev_cmd_list) || - unlikely(kthread_should_stop()); - return res; -} - -static int fileio_cmd_thread(void *arg) -{ - struct scst_fileio_tgt_dev *ftgt_dev = (struct scst_fileio_tgt_dev*)arg; - - TRACE_ENTRY(); - - set_user_nice(current, 10); - current->flags |= PF_NOFREEZE; - - spin_lock_bh(&ftgt_dev->fdev_lock); - while (!kthread_should_stop()) { - wait_queue_t wait; - struct scst_cmd *cmd; - init_waitqueue_entry(&wait, current); - - if (!test_cmd_list(ftgt_dev)) { - add_wait_queue_exclusive(&ftgt_dev->fdev_waitQ, &wait); - for (;;) { - set_current_state(TASK_INTERRUPTIBLE); - if (test_cmd_list(ftgt_dev)) - break; - spin_unlock_bh(&ftgt_dev->fdev_lock); - schedule(); - spin_lock_bh(&ftgt_dev->fdev_lock); - } - set_current_state(TASK_RUNNING); - remove_wait_queue(&ftgt_dev->fdev_waitQ, &wait); - } - - while (!list_empty(&ftgt_dev->fdev_cmd_list)) { - cmd = list_entry(ftgt_dev->fdev_cmd_list.next, - typeof(*cmd), fileio_cmd_list_entry); - cmd->fileio_in_list = 0; - list_del(&cmd->fileio_cmd_list_entry); - spin_unlock_bh(&ftgt_dev->fdev_lock); - fileio_do_job(cmd); - spin_lock_bh(&ftgt_dev->fdev_lock); - } - } - spin_unlock_bh(&ftgt_dev->fdev_lock); - - /* - * If kthread_should_stop() is true, we are guaranteed to be in - * suspended activity state, so fdev_cmd_list must be empty. - */ - sBUG_ON(!list_empty(&ftgt_dev->fdev_cmd_list)); - - TRACE_EXIT(); - return 0; -} - -static int fileio_attach_tgt(struct scst_tgt_dev *tgt_dev) -{ - struct scst_fileio_dev *virt_dev = - (struct scst_fileio_dev *)tgt_dev->acg_dev->dev->dh_priv; - struct scst_fileio_tgt_dev *ftgt_dev; - int res = 0; - - TRACE_ENTRY(); - - ftgt_dev = kzalloc(sizeof(*ftgt_dev), GFP_KERNEL); - if (ftgt_dev == NULL) { - TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of per-session " - "virtual device failed"); - res = -ENOMEM; - goto out; - } - - spin_lock_init(&ftgt_dev->fdev_lock); - INIT_LIST_HEAD(&ftgt_dev->fdev_cmd_list); - init_waitqueue_head(&ftgt_dev->fdev_waitQ); - ftgt_dev->virt_dev = virt_dev; - - if (!virt_dev->cdrom_empty && !virt_dev->nullio) { - ftgt_dev->fd = fileio_open(virt_dev); - if (IS_ERR(ftgt_dev->fd)) { - res = PTR_ERR(ftgt_dev->fd); - PRINT_ERROR_PR("filp_open(%s) returned an error %d", - virt_dev->file_name, res); - goto out_free; - } - } else - ftgt_dev->fd = NULL; - - /* - * Only ONE thread must be run here, otherwise the commands could - * be executed out of order !! - */ - ftgt_dev->cmd_thread = kthread_run(fileio_cmd_thread, ftgt_dev, "scst_fileio"); - if (IS_ERR(ftgt_dev->cmd_thread)) { - PRINT_ERROR_PR("kthread_run failed to create %s", "scst_fileio"); - res = PTR_ERR(ftgt_dev->cmd_thread); - goto out_free_close; - } - - tgt_dev->dh_priv = ftgt_dev; - - down(&virt_dev->ftgt_list_mutex); - list_add_tail(&ftgt_dev->ftgt_list_entry, - &virt_dev->ftgt_list); - up(&virt_dev->ftgt_list_mutex); - -out: - TRACE_EXIT_RES(res); - return res; - -out_free_close: - if (ftgt_dev->fd) - filp_close(ftgt_dev->fd, NULL); - -out_free: - kfree(ftgt_dev); - goto out; -} - -static void fileio_detach_tgt(struct scst_tgt_dev *tgt_dev) -{ - struct scst_fileio_tgt_dev *ftgt_dev = - (struct scst_fileio_tgt_dev *)tgt_dev->dh_priv; - struct scst_fileio_dev *virt_dev = - (struct scst_fileio_dev *)tgt_dev->acg_dev->dev->dh_priv; - - TRACE_ENTRY(); - - down(&virt_dev->ftgt_list_mutex); - list_del(&ftgt_dev->ftgt_list_entry); - up(&virt_dev->ftgt_list_mutex); - - kthread_stop(ftgt_dev->cmd_thread); - - if (ftgt_dev->fd) - filp_close(ftgt_dev->fd, NULL); - - if (ftgt_dev->iv != NULL) - kfree(ftgt_dev->iv); - - kfree(ftgt_dev); - - tgt_dev->dh_priv = NULL; + if (likely(thr != NULL)) + scst_thr_data_put(&thr->hdr); TRACE_EXIT(); + return SCST_EXEC_COMPLETED; } /******************************************************************** @@ -872,111 +869,6 @@ static int disk_fileio_parse(struct scst_cmd *cmd, return res; } -static inline void fileio_queue_cmd(struct scst_cmd *cmd) -{ - struct scst_fileio_tgt_dev *ftgt_dev = - (struct scst_fileio_tgt_dev *)cmd->tgt_dev->dh_priv; - spin_lock_bh(&ftgt_dev->fdev_lock); - TRACE_DBG("Pushing cmd %p to IO thread", cmd); - list_add_tail(&cmd->fileio_cmd_list_entry, - &ftgt_dev->fdev_cmd_list); - cmd->fileio_in_list = 1; - spin_unlock_bh(&ftgt_dev->fdev_lock); - wake_up(&ftgt_dev->fdev_waitQ); -} - -/******************************************************************** - * Function: disk_fileio_exec - * - * Argument: - * - * Returns : always SCST_EXEC_COMPLETED, real status is in error condition - * in command - * - * Description: - ********************************************************************/ -static int disk_fileio_exec(struct scst_cmd *cmd) -{ - int delayed = 0; - int opcode = cmd->cdb[0]; - - TRACE_ENTRY(); - - cmd->status = 0; - cmd->msg_status = 0; - cmd->host_status = DID_OK; - cmd->driver_status = 0; - - /* - * !! - * Only commands that unsensible to the execution order could be - * performed here, in place. Other ones must be passed to the - * thread. - * !! - */ - switch (opcode) { - case INQUIRY: - fileio_exec_inquiry(cmd); - break; - case READ_CAPACITY: - fileio_exec_read_capacity(cmd); - break; - case SERVICE_ACTION_IN: - if ((cmd->cdb[1] & 0x1f) == SAI_READ_CAPACITY_16) - fileio_exec_read_capacity16(cmd); - else - goto def; - break; - case WRITE_VERIFY: - case WRITE_VERIFY_12: - case WRITE_VERIFY_16: - /* fall through */ - case WRITE_6: - case WRITE_10: - case WRITE_12: - case WRITE_16: - /* could move READ ONLY check up to here (currenlty in do_job()) */ - /* fall through */ - case MODE_SENSE: - case MODE_SENSE_10: - case MODE_SELECT: - case MODE_SELECT_10: - case READ_6: - case READ_10: - case READ_12: - case READ_16: - case SYNCHRONIZE_CACHE: - case VERIFY_6: - case VERIFY: - case VERIFY_12: - case VERIFY_16: - case START_STOP: - case RESERVE: - case RESERVE_10: - case RELEASE: - case RELEASE_10: - fileio_queue_cmd(cmd); - delayed = 1; - break; - case TEST_UNIT_READY: - break; - case REPORT_LUNS: -def: - default: - TRACE_DBG("Invalid opcode 0x%02x", opcode); - scst_set_cmd_error(cmd, - SCST_LOAD_SENSE(scst_sense_invalid_opcode)); - } - - if (!delayed) { - cmd->completed = 1; - cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT); - } - - TRACE_EXIT(); - return SCST_EXEC_COMPLETED; -} - /******************************************************************** * Function: cdrom_fileio_parse * @@ -1057,7 +949,6 @@ static int cdrom_fileio_parse(struct scst_cmd *cmd, ********************************************************************/ static int cdrom_fileio_exec(struct scst_cmd *cmd) { - int delayed = 0; int opcode = cmd->cdb[0]; struct scst_fileio_dev *virt_dev = (struct scst_fileio_dev *)cmd->dev->dh_priv; @@ -1073,84 +964,33 @@ static int cdrom_fileio_exec(struct scst_cmd *cmd) TRACE_DBG("%s", "CDROM empty"); scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_not_ready)); - goto out; + goto out_complete; } - /* - * No protection is necessary, because media_changed set only - * in suspended state and exec() is serialized - */ if (virt_dev->media_changed && (cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REQUEST_SENSE) && (cmd->cdb[0] != REPORT_LUNS)) { - virt_dev->media_changed = 0; - TRACE_DBG("%s", "Reporting media changed"); - scst_set_cmd_error(cmd, - SCST_LOAD_SENSE(scst_sense_medium_changed_UA)); - goto out; + spin_lock(&virt_dev->flags_lock); + if (virt_dev->media_changed) { + virt_dev->media_changed = 0; + TRACE_DBG("%s", "Reporting media changed"); + scst_set_cmd_error(cmd, + SCST_LOAD_SENSE(scst_sense_medium_changed_UA)); + spin_unlock(&virt_dev->flags_lock); + goto out_complete; + } + spin_unlock(&virt_dev->flags_lock); } - /* - * !! - * Only commands that unsensible to the execution order could be - * performed here, in place. Other ones must be passed to the - * thread. - * !! - */ - - switch (opcode) { - case INQUIRY: - fileio_exec_inquiry(cmd); - break; - case READ_CAPACITY: - fileio_exec_read_capacity(cmd); - break; - case WRITE_VERIFY: - case WRITE_VERIFY_12: - case WRITE_VERIFY_16: - /* fall through */ - case MODE_SENSE: - case MODE_SENSE_10: - case MODE_SELECT: - case MODE_SELECT_10: - case READ_6: - case READ_10: - case READ_12: - case READ_16: - case WRITE_6: - case WRITE_10: - case WRITE_12: - case WRITE_16: - case VERIFY_6: - case VERIFY: - case VERIFY_12: - case VERIFY_16: - case START_STOP: - case RESERVE: - case RESERVE_10: - case RELEASE: - case RELEASE_10: - case ALLOW_MEDIUM_REMOVAL: - case READ_TOC: - fileio_queue_cmd(cmd); - delayed = 1; - break; - case TEST_UNIT_READY: - break; - case REPORT_LUNS: - default: - TRACE_DBG("Invalid opcode 0x%02x", opcode); - scst_set_cmd_error(cmd, - SCST_LOAD_SENSE(scst_sense_invalid_opcode)); - } + fileio_do_job(cmd); out: - if (!delayed) { - cmd->completed = 1; - cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT); - } - TRACE_EXIT(); return SCST_EXEC_COMPLETED; + +out_complete: + cmd->completed = 1; + cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT); + goto out; } static void fileio_exec_inquiry(struct scst_cmd *cmd) @@ -1566,40 +1406,17 @@ out: static int fileio_set_wt(struct scst_fileio_dev *virt_dev, int wt) { int res = 0; - struct scst_fileio_tgt_dev *ftgt_dev; - struct file *fd; TRACE_ENTRY(); if ((virt_dev->wt_flag == wt) || virt_dev->nullio) goto out; + spin_lock(&virt_dev->flags_lock); virt_dev->wt_flag = wt; + spin_unlock(&virt_dev->flags_lock); - scst_suspend_activity(); - - down(&virt_dev->ftgt_list_mutex); - list_for_each_entry(ftgt_dev, &virt_dev->ftgt_list, - ftgt_list_entry) - { - fd = fileio_open(virt_dev); - if (IS_ERR(fd)) { - res = PTR_ERR(fd); - PRINT_ERROR_PR("filp_open(%s) returned an error %d, " - "unable to change the cache mode", - virt_dev->file_name, res); - up(&virt_dev->ftgt_list_mutex); - res = 0; /* ?? ToDo */ - goto out_resume; - } - if (ftgt_dev->fd) - filp_close(ftgt_dev->fd, NULL); - ftgt_dev->fd = fd; - } - up(&virt_dev->ftgt_list_mutex); - -out_resume: - scst_resume_activity(); + scst_dev_del_all_thr_data(virt_dev->dev); out: TRACE_EXIT_RES(res); @@ -1885,10 +1702,7 @@ static void fileio_exec_prevent_allow_medium_removal(struct scst_cmd *cmd) TRACE_DBG("PERSIST/PREVENT 0x%02x", cmd->cdb[4]); - /* - * No protection here, because in cdrom_fileio_change() the - * activity is suspended and exec() is serialized - */ + spin_lock(&virt_dev->flags_lock); if (cmd->dev->handler->type == TYPE_ROM) virt_dev->prevent_allow_medium_removal = cmd->cdb[4] & 0x01 ? 1 : 0; @@ -1898,21 +1712,22 @@ static void fileio_exec_prevent_allow_medium_removal(struct scst_cmd *cmd) scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode)); } + spin_unlock(&virt_dev->flags_lock); return; } -static int fileio_fsync(struct scst_fileio_tgt_dev *ftgt_dev, +static int fileio_fsync(struct scst_fileio_thr *thr, loff_t loff, loff_t len, struct scst_cmd *cmd) { int res = 0; - struct file *file = ftgt_dev->fd; + struct file *file = thr->fd; struct inode *inode = file->f_dentry->d_inode; struct address_space *mapping = file->f_mapping; TRACE_ENTRY(); - if (ftgt_dev->virt_dev->nv_cache) + if (thr->virt_dev->nv_cache) goto out; res = sync_page_range(inode, mapping, loff, len); @@ -1932,25 +1747,25 @@ out: } static struct iovec *fileio_alloc_iv(struct scst_cmd *cmd, - struct scst_fileio_tgt_dev *ftgt_dev) + struct scst_fileio_thr *thr) { int iv_count; iv_count = scst_get_buf_count(cmd); - if (iv_count > ftgt_dev->iv_count) { - if (ftgt_dev->iv != NULL) - kfree(ftgt_dev->iv); - ftgt_dev->iv = kmalloc(sizeof(*ftgt_dev->iv) * iv_count, GFP_KERNEL); - if (ftgt_dev->iv == NULL) { + if (iv_count > thr->iv_count) { + if (thr->iv != NULL) + kfree(thr->iv); + thr->iv = kmalloc(sizeof(*thr->iv) * iv_count, GFP_KERNEL); + if (thr->iv == NULL) { PRINT_ERROR_PR("Unable to allocate iv (%d)", iv_count); scst_set_busy(cmd); goto out; } - ftgt_dev->iv_count = iv_count; + thr->iv_count = iv_count; } out: - return ftgt_dev->iv; + return thr->iv; } /* @@ -1995,7 +1810,8 @@ ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov, } #endif -static void fileio_exec_read(struct scst_cmd *cmd, loff_t loff) +static void fileio_exec_read(struct scst_cmd *cmd, + struct scst_fileio_thr *thr, loff_t loff) { mm_segment_t old_fs; loff_t err; @@ -2003,15 +1819,13 @@ static void fileio_exec_read(struct scst_cmd *cmd, loff_t loff) uint8_t *address; struct scst_fileio_dev *virt_dev = (struct scst_fileio_dev *)cmd->dev->dh_priv; - struct scst_fileio_tgt_dev *ftgt_dev = - (struct scst_fileio_tgt_dev *)cmd->tgt_dev->dh_priv; - struct file *fd = ftgt_dev->fd; + struct file *fd = thr->fd; struct iovec *iv; int iv_count, i; TRACE_ENTRY(); - iv = fileio_alloc_iv(cmd, ftgt_dev); + iv = fileio_alloc_iv(cmd, thr); if (iv == NULL) goto out; @@ -2085,7 +1899,8 @@ out: return; } -static void fileio_exec_write(struct scst_cmd *cmd, loff_t loff) +static void fileio_exec_write(struct scst_cmd *cmd, + struct scst_fileio_thr *thr, loff_t loff) { mm_segment_t old_fs; loff_t err; @@ -2093,15 +1908,13 @@ static void fileio_exec_write(struct scst_cmd *cmd, loff_t loff) uint8_t *address; struct scst_fileio_dev *virt_dev = (struct scst_fileio_dev *)cmd->dev->dh_priv; - struct scst_fileio_tgt_dev *ftgt_dev = - (struct scst_fileio_tgt_dev *)cmd->tgt_dev->dh_priv; - struct file *fd = ftgt_dev->fd; + struct file *fd = thr->fd; struct iovec *iv, *eiv; int iv_count, eiv_count; TRACE_ENTRY(); - iv = fileio_alloc_iv(cmd, ftgt_dev); + iv = fileio_alloc_iv(cmd, thr); if (iv == NULL) goto out; @@ -2209,7 +2022,8 @@ out: return; } -static void fileio_exec_verify(struct scst_cmd *cmd, loff_t loff) +static void fileio_exec_verify(struct scst_cmd *cmd, + struct scst_fileio_thr *thr, loff_t loff) { mm_segment_t old_fs; loff_t err; @@ -2218,14 +2032,12 @@ static void fileio_exec_verify(struct scst_cmd *cmd, loff_t loff) int compare; struct scst_fileio_dev *virt_dev = (struct scst_fileio_dev *)cmd->dev->dh_priv; - struct scst_fileio_tgt_dev *ftgt_dev = - (struct scst_fileio_tgt_dev *)cmd->tgt_dev->dh_priv; - struct file *fd = ftgt_dev->fd; + struct file *fd = thr->fd; uint8_t *mem_verify = NULL; TRACE_ENTRY(); - if (fileio_fsync(ftgt_dev, loff, cmd->bufflen, cmd) != 0) + if (fileio_fsync(thr, loff, cmd->bufflen, cmd) != 0) goto out; /* @@ -2325,35 +2137,6 @@ out: return; } -/* No locks supposed to be held, thread context */ -static int fileio_task_mgmt_fn(struct scst_mgmt_cmd *mcmd, - struct scst_tgt_dev *tgt_dev) -{ - int res = SCST_DEV_TM_NOT_COMPLETED; - - TRACE_ENTRY(); - - if (mcmd->fn == SCST_ABORT_TASK) { - struct scst_cmd *cmd_to_abort = mcmd->cmd_to_abort; - struct scst_fileio_tgt_dev *ftgt_dev = - (struct scst_fileio_tgt_dev *)cmd_to_abort->tgt_dev->dh_priv; - - spin_lock_bh(&ftgt_dev->fdev_lock); - if (cmd_to_abort->fileio_in_list) { - TRACE(TRACE_MGMT, "Aborting cmd %p and moving it to " - "the queue head", cmd_to_abort); - list_del(&cmd_to_abort->fileio_cmd_list_entry); - list_add(&cmd_to_abort->fileio_cmd_list_entry, - &ftgt_dev->fdev_cmd_list); - wake_up(&ftgt_dev->fdev_waitQ); - } - spin_unlock_bh(&ftgt_dev->fdev_lock); - } - - TRACE_EXIT_RES(res); - return res; -} - static inline struct scst_fileio_dev *fileio_alloc_dev(void) { struct scst_fileio_dev *dev; @@ -2363,8 +2146,7 @@ static inline struct scst_fileio_dev *fileio_alloc_dev(void) "device failed"); goto out; } - INIT_LIST_HEAD(&dev->ftgt_list); - init_MUTEX(&dev->ftgt_list_mutex); + spin_lock_init(&dev->flags_lock); out: return dev; } @@ -2797,7 +2579,6 @@ out: static int cdrom_fileio_change(char *p, char *name) { struct file *fd; - struct scst_fileio_tgt_dev *ftgt_dev; loff_t err; mm_segment_t old_fs; struct scst_fileio_dev *virt_dev, *vv; @@ -2908,24 +2689,7 @@ static int cdrom_fileio_change(char *p, char *name) if (!virt_dev->cdrom_empty) virt_dev->media_changed = 1; - down(&virt_dev->ftgt_list_mutex); - list_for_each_entry(ftgt_dev, &virt_dev->ftgt_list, ftgt_list_entry) { - if (!virt_dev->cdrom_empty && !virt_dev->nullio) { - fd = fileio_open(virt_dev); - if (IS_ERR(fd)) { - res = PTR_ERR(fd); - PRINT_ERROR_PR("filp_open(%s) returned an error %d, " - "closing the device", virt_dev->file_name, res); - up(&virt_dev->ftgt_list_mutex); - goto out_err_resume; - } - } else - fd = NULL; - if (ftgt_dev->fd) - filp_close(ftgt_dev->fd, NULL); - ftgt_dev->fd = fd; - } - up(&virt_dev->ftgt_list_mutex); + scst_dev_del_all_thr_data(virt_dev->dev); if (!virt_dev->cdrom_empty) { PRINT_INFO_PR("Changed SCSI target virtual cdrom %s " @@ -2958,13 +2722,6 @@ out_free_resume: virt_dev->file_name = old_fn; kfree(fn); goto out_resume; - -out_err_resume: - virt_dev->file_name = old_fn; - kfree(fn); - scst_resume_activity(); - cdrom_fileio_close(name); - goto out; } /* @@ -3134,6 +2891,7 @@ static int __init init_scst_fileio(struct scst_dev_type *devtype) TRACE_ENTRY(); devtype->module = THIS_MODULE; + res = scst_register_virtual_dev_driver(devtype); if (res < 0) goto out; @@ -3197,9 +2955,18 @@ static void __exit exit_scst_fileio(struct scst_dev_type *devtype, static int __init init_scst_fileio_driver(void) { int res; + + fileio_thr_cachep = kmem_cache_create("fileio_thr_data", + sizeof(struct scst_fileio_thr), 0, FILEIO_SLAB_FLAGS, NULL, + NULL); + if (fileio_thr_cachep == NULL) { + res = -ENOMEM; + goto out; + } + res = init_scst_fileio(&disk_devtype_fileio); if (res != 0) - goto out; + goto out_free_slab; res = init_scst_fileio(&cdrom_devtype_fileio); if (res != 0) @@ -3210,6 +2977,9 @@ out: out_err: exit_scst_fileio(&disk_devtype_fileio, &disk_fileio_dev_list); + +out_free_slab: + kmem_cache_destroy(fileio_thr_cachep); goto out; } @@ -3217,6 +2987,7 @@ static void __exit exit_scst_fileio_driver(void) { exit_scst_fileio(&disk_devtype_fileio, &disk_fileio_dev_list); exit_scst_fileio(&cdrom_devtype_fileio, &cdrom_fileio_dev_list); + kmem_cache_destroy(fileio_thr_cachep); } module_init(init_scst_fileio_driver); diff --git a/scst/src/scst.c b/scst/src/scst.c index d8c6fec19..e996a7219 100644 --- a/scst/src/scst.c +++ b/scst/src/scst.c @@ -445,6 +445,8 @@ static int scst_register_device(struct scsi_device *scsidp) goto out_up; } + dev->type = scsidp->type; + dev->rq_disk = alloc_disk(1); if (dev->rq_disk == NULL) { res = -ENOMEM; @@ -594,6 +596,7 @@ int scst_register_virtual_device(struct scst_dev_type *dev_handler, goto out; } + dev->type = dev_handler->type; dev->scsi_dev = NULL; dev->virt_name = dev_name; @@ -1274,7 +1277,7 @@ static int __init init_scst(void) scst_scsi_op_list_init(); - for (i = 0; i < sizeof(scst_tasklets)/sizeof(scst_tasklets[0]); i++) { + for (i = 0; i < ARRAY_SIZE(scst_tasklets); i++) { spin_lock_init(&scst_tasklets[i].tasklet_lock); INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list); tasklet_init(&scst_tasklets[i].tasklet, (void*)scst_cmd_tasklet, @@ -1469,6 +1472,12 @@ EXPORT_SYMBOL(scst_put); EXPORT_SYMBOL(scst_alloc); EXPORT_SYMBOL(scst_free); +/* Tgt_dev's threads local storage */ +EXPORT_SYMBOL(scst_add_thr_data); +EXPORT_SYMBOL(scst_del_all_thr_data); +EXPORT_SYMBOL(scst_dev_del_all_thr_data); +EXPORT_SYMBOL(scst_find_thr_data); + /* * Other Commands */ diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index 9afc0df2b..ecf0ea017 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -310,6 +311,74 @@ out: return res; } +static int scst_create_tgt_threads(struct scst_tgt_dev *tgt_dev) +{ + int res = 0; + static atomic_t major = ATOMIC_INIT(0); + char nm[12]; + + TRACE_ENTRY(); + + if ( !tgt_dev->dev->handler->dedicated_thread) + goto out; + + spin_lock_init(&tgt_dev->cmd_lists.cmd_list_lock); + INIT_LIST_HEAD(&tgt_dev->cmd_lists.active_cmd_list); + init_waitqueue_head(&tgt_dev->cmd_lists.cmd_list_waitQ); + + /* + * Only ONE thread must be run here, otherwise the commands could + * be executed out of order !! + */ + + strncpy(nm, tgt_dev->dev->handler->name, ARRAY_SIZE(nm)-1); + nm[ARRAY_SIZE(nm)-1] = '\0'; + tgt_dev->thread = kthread_run(scst_cmd_thread, &tgt_dev->cmd_lists, + "%sd%d", nm, atomic_inc_return(&major)); + if (IS_ERR(tgt_dev->thread)) { + res = PTR_ERR(tgt_dev->thread); + PRINT_ERROR_PR("kthread_create() failed: %d", res); + tgt_dev->thread = NULL; + goto out; + } + + down(&scst_suspend_mutex); + list_add_tail(&tgt_dev->cmd_lists.lists_list_entry, + &scst_cmd_lists_list); + up(&scst_suspend_mutex); + + tgt_dev->p_cmd_lists = &tgt_dev->cmd_lists; + +out: + TRACE_EXIT(); + return res; +} + +static void scst_stop_tgt_threads(struct scst_tgt_dev *tgt_dev) +{ + int rc; + + TRACE_ENTRY(); + + if (tgt_dev->thread == NULL) + goto out; + + rc = kthread_stop(tgt_dev->thread); + if (rc < 0) { + TRACE_MGMT_DBG("kthread_stop() failed: %d", rc); + } + + if (tgt_dev->p_cmd_lists == &tgt_dev->cmd_lists) { + down(&scst_suspend_mutex); + list_del(&tgt_dev->cmd_lists.lists_list_entry); + up(&scst_suspend_mutex); + } + +out: + TRACE_EXIT(); + return; +} + /* * No spin locks supposed to be held, scst_mutex - held. * The activity is suspended. @@ -319,7 +388,7 @@ static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess, { struct scst_tgt_dev *tgt_dev; struct scst_device *dev = acg_dev->dev; - int res; + int rc, i; TRACE_ENTRY(); @@ -337,11 +406,11 @@ static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess, memset(tgt_dev, 0, sizeof(*tgt_dev)); #endif + tgt_dev->dev = acg_dev->dev; + tgt_dev->lun = acg_dev->lun; tgt_dev->acg_dev = acg_dev; tgt_dev->sess = sess; atomic_set(&tgt_dev->cmd_count, 0); - atomic_set(&tgt_dev->curr_sn, 0); - tgt_dev->expected_sn = 1; tgt_dev->p_cmd_lists = &scst_main_cmd_lists; @@ -349,18 +418,26 @@ static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess, TRACE(TRACE_DEBUG, "host=%d, channel=%d, id=%d, lun=%d, " "SCST lun=%Ld", dev->scsi_dev->host->host_no, dev->scsi_dev->channel, dev->scsi_dev->id, - dev->scsi_dev->lun, (uint64_t)tgt_dev->acg_dev->lun); + dev->scsi_dev->lun, (uint64_t)tgt_dev->lun); } else { TRACE_MGMT_DBG("Virtual device SCST lun=%Ld", - (uint64_t)tgt_dev->acg_dev->lun); + (uint64_t)tgt_dev->lun); } spin_lock_init(&tgt_dev->tgt_dev_lock); INIT_LIST_HEAD(&tgt_dev->UA_list); + spin_lock_init(&tgt_dev->thr_data_lock); + INIT_LIST_HEAD(&tgt_dev->thr_data_list); spin_lock_init(&tgt_dev->sn_lock); INIT_LIST_HEAD(&tgt_dev->deferred_cmd_list); INIT_LIST_HEAD(&tgt_dev->skipped_sn_list); + INIT_LIST_HEAD(&tgt_dev->hq_cmd_list); + tgt_dev->expected_sn = 1; + tgt_dev->num_free_sn_slots = ARRAY_SIZE(tgt_dev->sn_slots); + tgt_dev->cur_sn_slot = &tgt_dev->sn_slots[0]; + for(i = 0; i < ARRAY_SIZE(tgt_dev->sn_slots); i++) + atomic_set(&tgt_dev->sn_slots[i], 0); if (dev->handler->parse_atomic && sess->tgt->tgtt->preprocessing_done_atomic) { @@ -394,15 +471,19 @@ static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess, tm_dbg_init_tgt_dev(tgt_dev, acg_dev); + rc = scst_create_tgt_threads(tgt_dev); + if (rc != 0) + goto out_free; + if (dev->handler && dev->handler->attach_tgt) { TRACE_DBG("Calling dev handler's attach_tgt(%p)", tgt_dev); - res = dev->handler->attach_tgt(tgt_dev); + rc = dev->handler->attach_tgt(tgt_dev); TRACE_DBG("%s", "Dev handler's attach_tgt() returned"); - if (res != 0) { + if (rc != 0) { PRINT_ERROR_PR("Device handler's %s attach_tgt() " - "failed: %d", dev->handler->name, res); - goto out_free; + "failed: %d", dev->handler->name, rc); + goto out_stop_free; } } @@ -417,6 +498,9 @@ out: TRACE_EXIT(); return tgt_dev; +out_stop_free: + scst_stop_tgt_threads(tgt_dev); + out_free: kmem_cache_free(scst_tgtd_cachep, tgt_dev); tgt_dev = NULL; @@ -431,7 +515,7 @@ static void scst_send_release(struct scst_tgt_dev *tgt_dev); */ void scst_reset_tgt_dev(struct scst_tgt_dev *tgt_dev, int nexus_loss) { - struct scst_device *dev = tgt_dev->acg_dev->dev; + struct scst_device *dev = tgt_dev->dev; if (dev->dev_reserved && !test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags)) @@ -467,7 +551,7 @@ void scst_reset_tgt_dev(struct scst_tgt_dev *tgt_dev, int nexus_loss) */ static void scst_free_tgt_dev(struct scst_tgt_dev *tgt_dev) { - struct scst_device *dev = tgt_dev->acg_dev->dev; + struct scst_device *dev = tgt_dev->dev; TRACE_ENTRY(); @@ -486,6 +570,8 @@ static void scst_free_tgt_dev(struct scst_tgt_dev *tgt_dev) TRACE_DBG("%s", "Dev handler's detach_tgt() returned"); } + scst_stop_tgt_threads(tgt_dev); + kmem_cache_free(scst_tgtd_cachep, tgt_dev); TRACE_EXIT(); @@ -864,10 +950,10 @@ static void scst_send_release(struct scst_tgt_dev *tgt_dev) TRACE_ENTRY(); - if (tgt_dev->acg_dev->dev->scsi_dev == NULL) + if (tgt_dev->dev->scsi_dev == NULL) goto out; - scsi_dev = tgt_dev->acg_dev->dev->scsi_dev; + scsi_dev = tgt_dev->dev->scsi_dev; req = scsi_allocate_request(scsi_dev, GFP_KERNEL); if (req == NULL) { @@ -888,7 +974,7 @@ static void scst_send_release(struct scst_tgt_dev *tgt_dev) req->sr_use_sg = 0; req->sr_bufflen = 0; req->sr_buffer = NULL; - req->sr_request->rq_disk = tgt_dev->acg_dev->dev->rq_disk; + req->sr_request->rq_disk = tgt_dev->dev->rq_disk; req->sr_sense_buffer[0] = 0; TRACE(TRACE_DEBUG | TRACE_SCSI, "Sending RELEASE req %p to SCSI " @@ -910,10 +996,10 @@ static void scst_send_release(struct scst_tgt_dev *tgt_dev) TRACE_ENTRY(); - if (tgt_dev->acg_dev->dev->scsi_dev == NULL) + if (tgt_dev->dev->scsi_dev == NULL) goto out; - scsi_dev = tgt_dev->acg_dev->dev->scsi_dev; + scsi_dev = tgt_dev->dev->scsi_dev; memset(cdb, 0, sizeof(cdb)); cdb[0] = RELEASE; @@ -1076,7 +1162,7 @@ struct scst_cmd *scst_alloc_cmd(int gfp_mask) atomic_set(&cmd->cmd_ref, 1); cmd->cmd_lists = &scst_main_cmd_lists; - cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED; + cmd->queue_type = SCST_CMD_QUEUE_SIMPLE; cmd->timeout = SCST_DEFAULT_TIMEOUT; cmd->retries = 1; cmd->data_len = -1; @@ -1156,19 +1242,18 @@ void scst_free_cmd(struct scst_cmd *cmd) if (likely(cmd->tgt_dev != NULL)) { #ifdef EXTRACHECKS - if (cmd->sent_to_midlev == 0) { - PRINT_ERROR_PR("Finishing not executed cmd (opcode %d, " - "target %s, lun %Ld, sn %d, expected_sn %d)", - cmd->cdb[0], cmd->tgtt->name, (uint64_t)cmd->lun, + if (unlikely(!cmd->sent_to_midlev)) { + PRINT_ERROR_PR("Finishing not executed cmd %p (opcode " + "%d, target %s, lun %Ld, sn %d, expected_sn %d)", + cmd, cmd->cdb[0], cmd->tgtt->name, (uint64_t)cmd->lun, cmd->sn, cmd->tgt_dev->expected_sn); - scst_inc_expected_sn_unblock(cmd->tgt_dev, cmd); + scst_unblock_deferred(cmd->tgt_dev, cmd); } #endif if (unlikely(cmd->out_of_sn)) { - TRACE(TRACE_SCSI_SERIALIZING, "Out of SN " - "cmd %p (tag %d, sn %d), destroy=%d", cmd, - cmd->tag, cmd->sn, destroy); + TRACE_SN("Out of SN cmd %p (tag %d, sn %d), " + "destroy=%d", cmd, cmd->tag, cmd->sn, destroy); destroy = test_and_set_bit(SCST_CMD_CAN_BE_DESTROYED, &cmd->cmd_flags); } @@ -1751,11 +1836,12 @@ void scst_process_reset(struct scst_device *dev, /* Clear RESERVE'ation, if necessary */ if (dev->dev_reserved) { + /* Either scst_mutex held or exclude_cmd non-NULL */ list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, dev_tgt_dev_list_entry) { TRACE(TRACE_MGMT, "Clearing RESERVE'ation for tgt_dev " - "lun %d", tgt_dev->acg_dev->lun); + "lun %d", tgt_dev->lun); clear_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags); } @@ -1791,7 +1877,7 @@ void scst_process_reset(struct scst_device *dev, continue; if ((cmd->tgt_dev == tgt_dev) || ((cmd->tgt_dev == NULL) && - (cmd->lun == tgt_dev->acg_dev->lun))) { + (cmd->lun == tgt_dev->lun))) { scst_abort_cmd(cmd, mcmd, (tgt_dev->sess != originator), 0); } @@ -1964,7 +2050,7 @@ void scst_free_all_UA(struct scst_tgt_dev *tgt_dev) list_for_each_entry_safe(UA_entry, t, &tgt_dev->UA_list, UA_list_entry) { TRACE_MGMT_DBG("Clearing UA for tgt_dev lun %d", - tgt_dev->acg_dev->lun); + tgt_dev->lun); list_del(&UA_entry->UA_list_entry); kfree(UA_entry); } @@ -1975,55 +2061,203 @@ void scst_free_all_UA(struct scst_tgt_dev *tgt_dev) return; } -struct scst_cmd *__scst_check_deferred_commands(struct scst_tgt_dev *tgt_dev, - int expected_sn) +/* sn_lock supposed to be held and IRQ off */ +static inline int __scst_check_hq_cmd(struct scst_cmd *cmd) { - struct scst_cmd *cmd = NULL, *tcmd; + struct scst_tgt_dev *tgt_dev = cmd->tgt_dev; + struct scst_cmd *hq; + int res; - if (tgt_dev->def_cmd_count == 0) - goto out; + TRACE_ENTRY(); - spin_lock_bh(&tgt_dev->sn_lock); - -restart: - list_for_each_entry(tcmd, &tgt_dev->deferred_cmd_list, - sn_cmd_list_entry) { - if (tcmd->sn == expected_sn) { - TRACE(TRACE_SCSI_SERIALIZING, - "Deferred command sn %d found", tcmd->sn); - tgt_dev->def_cmd_count--; - list_del(&tcmd->sn_cmd_list_entry); - cmd = tcmd; - goto out_unlock; - } + /* According to SAM, the latest HQ cmd shall pass first */ + hq = list_entry(tgt_dev->hq_cmd_list.next, typeof(*hq), + sn_cmd_list_entry); + if ((cmd == hq) && !test_bit(SCST_TGT_DEV_HQ_ACTIVE, + &tgt_dev->tgt_dev_flags)) { + TRACE_SN("Passing HQ cmd %p", cmd); + res = 1; + list_del(&cmd->sn_cmd_list_entry); + set_bit(SCST_TGT_DEV_HQ_ACTIVE, &tgt_dev->tgt_dev_flags); + } else { + TRACE_SN("Defer HQ cmd %p", cmd); + res = 0; + cmd->hq_deferred = 1; + tgt_dev->def_cmd_count++; } - list_for_each_entry(tcmd, &tgt_dev->skipped_sn_list, - sn_cmd_list_entry) { - if (tcmd->sn == expected_sn) { - /* - * !! At this point any pointer in tcmd, except !! - * !! sn_cmd_list_entry, could be already destroyed !! - */ - TRACE(TRACE_SCSI_SERIALIZING, - "cmd %p (tag %d) with skipped sn %d found", tcmd, - tcmd->tag, tcmd->sn); - tgt_dev->def_cmd_count--; - list_del(&tcmd->sn_cmd_list_entry); - if (test_and_set_bit(SCST_CMD_CAN_BE_DESTROYED, - &tcmd->cmd_flags)) { - scst_destroy_put_cmd(tcmd); + TRACE_EXIT_RES(res); + return res; +} + +int scst_check_hq_cmd(struct scst_cmd *cmd) +{ + struct scst_tgt_dev *tgt_dev = cmd->tgt_dev; + int res; + + spin_lock_irq(&tgt_dev->sn_lock); + res = __scst_check_hq_cmd(cmd); + spin_unlock_irq(&tgt_dev->sn_lock); + + return res; +} + +/* No locks */ +struct scst_cmd *__scst_check_deferred_commands(struct scst_tgt_dev *tgt_dev) +{ + struct scst_cmd *res = NULL, *cmd, *t; + int expected_sn = tgt_dev->expected_sn; + + spin_lock_irq(&tgt_dev->sn_lock); + + if (unlikely(test_bit(SCST_TGT_DEV_HQ_ACTIVE, &tgt_dev->tgt_dev_flags))) { + if (!list_empty(&tgt_dev->hq_cmd_list)) { + int rc; + cmd = list_entry(tgt_dev->hq_cmd_list.next, + typeof(*cmd), sn_cmd_list_entry); + if (cmd->hq_deferred) { + TRACE_SN("Releasing deferred HQ cmd %p", cmd); + tgt_dev->def_cmd_count--; + cmd->hq_deferred = 0; + res = cmd; + /* + * Since __scst_check_hq_cmd() is inline, a lot + * of code should be optimized out + */ + clear_bit(SCST_TGT_DEV_HQ_ACTIVE, + &tgt_dev->tgt_dev_flags); + rc = __scst_check_hq_cmd(res); + EXTRACHECKS_BUG_ON(rc != 1); + goto out_unlock; } - expected_sn = __scst_inc_expected_sn(tgt_dev); + } + TRACE_SN("Turning OFF hq_cmd_active (tgt_dev %p)", + tgt_dev); + clear_bit(SCST_TGT_DEV_HQ_ACTIVE, &tgt_dev->tgt_dev_flags); + } + +restart: + list_for_each_entry_safe(cmd, t, &tgt_dev->deferred_cmd_list, + sn_cmd_list_entry) { + if (cmd->sn == expected_sn) { + TRACE_SN("Deferred command %p (sn %d) found", + cmd, cmd->sn); + tgt_dev->def_cmd_count--; + list_del(&cmd->sn_cmd_list_entry); + if (res == NULL) + res = cmd; + else { + spin_lock(&cmd->cmd_lists->cmd_list_lock); + TRACE_SN("Adding cmd %p to active cmd list", + cmd); + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + wake_up(&cmd->cmd_lists->cmd_list_waitQ); + spin_unlock(&cmd->cmd_lists->cmd_list_lock); + } + } + } + if (res != NULL) + goto out_unlock; + + list_for_each_entry(cmd, &tgt_dev->skipped_sn_list, + sn_cmd_list_entry) { + if (cmd->sn == expected_sn) { + atomic_t *slot = cmd->sn_slot; + /* + * !! At this point any pointer in cmd, except !! + * !! sn_slot and sn_cmd_list_entry, could be !! + * !! already destroyed !! + */ + TRACE_SN("cmd %p (tag %d) with skipped sn %d found", + cmd, cmd->tag, cmd->sn); + tgt_dev->def_cmd_count--; + list_del(&cmd->sn_cmd_list_entry); + spin_unlock_irq(&tgt_dev->sn_lock); + EXTRACHECKS_BUG_ON(cmd->head_of_queue); + if (test_and_set_bit(SCST_CMD_CAN_BE_DESTROYED, + &cmd->cmd_flags)) { + scst_destroy_put_cmd(cmd); + } + scst_inc_expected_sn(tgt_dev, slot); + expected_sn = tgt_dev->expected_sn; + spin_lock_irq(&tgt_dev->sn_lock); goto restart; } } out_unlock: - spin_unlock_bh(&tgt_dev->sn_lock); + spin_unlock_irq(&tgt_dev->sn_lock); + return res; +} -out: - return cmd; +void scst_add_thr_data(struct scst_tgt_dev *tgt_dev, + struct scst_thr_data_hdr *data, + void (*free_fn) (struct scst_thr_data_hdr *data)) +{ + data->pid = current->pid; + atomic_set(&data->ref, 1); + EXTRACHECKS_BUG_ON(free_fn == NULL); + data->free_fn = free_fn; + spin_lock(&tgt_dev->thr_data_lock); + list_add_tail(&data->thr_data_list_entry, &tgt_dev->thr_data_list); + spin_unlock(&tgt_dev->thr_data_lock); +} + +void scst_del_all_thr_data(struct scst_tgt_dev *tgt_dev) +{ + spin_lock(&tgt_dev->thr_data_lock); + while (!list_empty(&tgt_dev->thr_data_list)) { + struct scst_thr_data_hdr *d = list_entry( + tgt_dev->thr_data_list.next, typeof(*d), + thr_data_list_entry); + list_del(&d->thr_data_list_entry); + spin_unlock(&tgt_dev->thr_data_lock); + scst_thr_data_put(d); + spin_lock(&tgt_dev->thr_data_lock); + } + spin_unlock(&tgt_dev->thr_data_lock); + return; +} + +void scst_dev_del_all_thr_data(struct scst_device *dev) +{ + struct scst_tgt_dev *tgt_dev; + + TRACE_ENTRY(); + + /* + * This is read-only function for dev->dev_tgt_dev_list, so + * suspending the activity isn't necessary. + */ + + down(&scst_mutex); + + list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, + dev_tgt_dev_list_entry) { + scst_del_all_thr_data(tgt_dev); + } + + up(&scst_mutex); + + TRACE_EXIT(); + return; +} + +struct scst_thr_data_hdr *scst_find_thr_data(struct scst_tgt_dev *tgt_dev) +{ + struct scst_thr_data_hdr *res = NULL, *d; + + spin_lock(&tgt_dev->thr_data_lock); + list_for_each_entry(d, &tgt_dev->thr_data_list, thr_data_list_entry) { + if (d->pid == current->pid) { + res = d; + scst_thr_data_get(res); + break; + } + } + spin_unlock(&tgt_dev->thr_data_lock); + return res; } /* No locks */ @@ -2041,7 +2275,7 @@ int scst_inc_on_dev_cmd(struct scst_cmd *cmd) if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) goto out_unlock; if (dev->block_count > 0) { - scst_dec_on_dev_cmd(cmd); + scst_dec_on_dev_cmd(cmd, 0); TRACE_MGMT_DBG("Delaying cmd %p due to blocking or serializing" "(tag %d, dev %p)", cmd, cmd->tag, dev); list_add_tail(&cmd->blocked_cmd_list_entry, @@ -2061,7 +2295,7 @@ repeat: goto out_unlock; barrier(); /* to reread block_count */ if (dev->block_count > 0) { - scst_dec_on_dev_cmd(cmd); + scst_dec_on_dev_cmd(cmd, 0); TRACE_MGMT_DBG("Delaying cmd %p due to blocking or " "serializing (tag %d, dev %p)", cmd, cmd->tag, dev); @@ -2118,7 +2352,7 @@ void scst_unblock_cmds(struct scst_device *dev) * Since only one cmd per time is being executed, expected_sn * can't change behind us, if the corresponding cmd is in * blocked_cmd_list, but we could be called before - * __scst_inc_expected_sn(). + * scst_inc_expected_sn(). */ if (likely(!cmd->internal) && likely(!cmd->retry)) { int expected_sn; @@ -2153,8 +2387,12 @@ void scst_unblock_cmds(struct scst_device *dev) list_del(&cmd->blocked_cmd_list_entry); TRACE_MGMT_DBG("Adding blocked cmd %p to active cmd list", cmd); spin_lock(&cmd->cmd_lists->cmd_list_lock); - list_add_tail(&cmd->cmd_list_entry, - &cmd->cmd_lists->active_cmd_list); + if (unlikely(cmd->head_of_queue)) + list_add(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + else + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); wake_up(&cmd->cmd_lists->cmd_list_waitQ); spin_unlock(&cmd->cmd_lists->cmd_list_lock); } @@ -2165,30 +2403,36 @@ void scst_unblock_cmds(struct scst_device *dev) return; } -static struct scst_cmd *__scst_inc_expected_sn_unblock( +static struct scst_cmd *__scst_unblock_deferred( struct scst_tgt_dev *tgt_dev, struct scst_cmd *out_of_sn_cmd) { struct scst_cmd *res = NULL; - if (out_of_sn_cmd->sn == tgt_dev->expected_sn) { - __scst_inc_expected_sn(tgt_dev); - res = scst_check_deferred_commands(tgt_dev, - tgt_dev->expected_sn); + if (out_of_sn_cmd->head_of_queue) { + TRACE_SN("HQ out_of_sn_cmd %p", out_of_sn_cmd); + spin_lock_irq(&out_of_sn_cmd->tgt_dev->sn_lock); + list_del(&out_of_sn_cmd->sn_cmd_list_entry); + spin_unlock_irq(&out_of_sn_cmd->tgt_dev->sn_lock); + res = scst_check_deferred_commands(tgt_dev); + } else if (out_of_sn_cmd->sn == tgt_dev->expected_sn) { + scst_inc_expected_sn(tgt_dev, out_of_sn_cmd->sn_slot); + res = scst_check_deferred_commands(tgt_dev); } else { out_of_sn_cmd->out_of_sn = 1; - spin_lock_bh(&tgt_dev->sn_lock); + spin_lock_irq(&tgt_dev->sn_lock); tgt_dev->def_cmd_count++; list_add_tail(&out_of_sn_cmd->sn_cmd_list_entry, &tgt_dev->skipped_sn_list); - TRACE(TRACE_SCSI_SERIALIZING, "out_of_sn_cmd %p with sn %d " - "added to skipped_sn_list (expected_sn %d)", - out_of_sn_cmd, out_of_sn_cmd->sn, tgt_dev->expected_sn); - spin_unlock_bh(&tgt_dev->sn_lock); + TRACE_SN("out_of_sn_cmd %p with sn %d added to skipped_sn_list " + "(expected_sn %d)", out_of_sn_cmd, out_of_sn_cmd->sn, + tgt_dev->expected_sn); + spin_unlock_irq(&tgt_dev->sn_lock); } + return res; } -void scst_inc_expected_sn_unblock(struct scst_tgt_dev *tgt_dev, +void scst_unblock_deferred(struct scst_tgt_dev *tgt_dev, struct scst_cmd *out_of_sn_cmd) { struct scst_cmd *cmd; @@ -2196,16 +2440,16 @@ void scst_inc_expected_sn_unblock(struct scst_tgt_dev *tgt_dev, TRACE_ENTRY(); if (out_of_sn_cmd->no_sn) { - TRACE(TRACE_SCSI_SERIALIZING, "cmd %p with no_sn", out_of_sn_cmd); + TRACE_SN("cmd %p with no_sn", out_of_sn_cmd); goto out; } - cmd = __scst_inc_expected_sn_unblock(tgt_dev, out_of_sn_cmd); + cmd = __scst_unblock_deferred(tgt_dev, out_of_sn_cmd); if (cmd != NULL) { unsigned long flags; spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags); - TRACE(TRACE_SCSI_SERIALIZING, "cmd %p with sn %d " - "added to the head of active cmd list", cmd, cmd->sn); + TRACE_SN("cmd %p with sn %d added to the head of active cmd " + "list", cmd, cmd->sn); list_add(&cmd->cmd_list_entry, &cmd->cmd_lists->active_cmd_list); wake_up(&cmd->cmd_lists->cmd_list_waitQ); spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags); @@ -2531,3 +2775,46 @@ int tm_dbg_is_release(void) return tm_dbg_flags.tm_dbg_release; } #endif /* DEBUG_TM */ + +#ifdef DEBUG_SN +void scst_check_debug_sn(struct scst_cmd *cmd) +{ + static spinlock_t lock = SPIN_LOCK_UNLOCKED; + static int type; + static int cnt; + unsigned long flags; + int old = cmd->queue_type; + + spin_lock_irqsave(&lock, flags); + + if (cnt == 0) { + if ((scst_random() % 1000) == 500) { + if ((scst_random() % 3) == 1) + type = SCST_CMD_QUEUE_HEAD_OF_QUEUE; + else + type = SCST_CMD_QUEUE_ORDERED; + do { + cnt = scst_random() % 10; + } while(cnt == 0); + } else + goto out_unlock; + } + + cmd->queue_type = type; + cnt--; + + if (((scst_random() % 1000) == 750)) + cmd->queue_type = SCST_CMD_QUEUE_ORDERED; + else if (((scst_random() % 1000) == 751)) + cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE; + else if (((scst_random() % 1000) == 752)) + cmd->queue_type = SCST_CMD_QUEUE_SIMPLE; + + TRACE_SN("DbgSN changed cmd %p: %d/%d (cnt %d)", cmd, old, + cmd->queue_type, cnt); + +out_unlock: + spin_unlock_irqrestore(&lock, flags); + return; +} +#endif /* DEBUG_SN */ diff --git a/scst/src/scst_priv.h b/scst/src/scst_priv.h index 078a1ff3d..a6b5b9a75 100644 --- a/scst/src/scst_priv.h +++ b/scst/src/scst_priv.h @@ -57,13 +57,20 @@ extern unsigned long scst_trace_flag; #define SCST_DEFAULT_LOG_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR | TRACE_PID | \ TRACE_FUNCTION | TRACE_SPECIAL | TRACE_MGMT | TRACE_MGMT_DEBUG | \ TRACE_RETRY) -#else + +#define TRACE_SN(args...) TRACE(TRACE_SCSI_SERIALIZING, args) + +#else /* DEBUG */ + # ifdef TRACING #define SCST_DEFAULT_LOG_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR | TRACE_PID | \ TRACE_SPECIAL) # else #define SCST_DEFAULT_LOG_FLAGS 0 # endif + +#define TRACE_SN(args...) + #endif /** @@ -92,7 +99,6 @@ extern unsigned long scst_trace_flag; #define SCST_CMD_STATE_RES_CONT_SAME 0 #define SCST_CMD_STATE_RES_CONT_NEXT 1 #define SCST_CMD_STATE_RES_NEED_THREAD 2 -#define SCST_CMD_STATE_RES_RESTART 3 /** Name of the "default" security group **/ #define SCST_DEFAULT_ACG_NAME "Default" @@ -148,6 +154,9 @@ extern struct list_head scst_dev_list; /* protected by scst_mutex */ extern struct list_head scst_dev_type_list; /* protected by scst_mutex */ extern wait_queue_head_t scst_dev_cmd_waitQ; +extern struct semaphore scst_suspend_mutex; +extern struct list_head scst_cmd_lists_list; /* protected by scst_suspend_mutex */ + extern struct list_head scst_acg_list; extern struct scst_acg *scst_default_acg; @@ -208,36 +217,23 @@ extern spinlock_t scst_temp_UA_lock; extern uint8_t scst_temp_UA[SCSI_SENSE_BUFFERSIZE]; extern struct scst_cmd *__scst_check_deferred_commands( - struct scst_tgt_dev *tgt_dev, int expected_sn); + struct scst_tgt_dev *tgt_dev); -/* Used to save the function call on th fast path */ +/* Used to save the function call on the fast path */ static inline struct scst_cmd *scst_check_deferred_commands( - struct scst_tgt_dev *tgt_dev, int expected_sn) + struct scst_tgt_dev *tgt_dev) { - if (tgt_dev->def_cmd_count == 0) + if ((tgt_dev->def_cmd_count == 0) && + likely(!test_bit(SCST_TGT_DEV_HQ_ACTIVE, &tgt_dev->tgt_dev_flags))) return NULL; else - return __scst_check_deferred_commands(tgt_dev, expected_sn); + return __scst_check_deferred_commands(tgt_dev); } -static inline int __scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev) -{ - /* - * No locks is needed, because only one thread at time can - * call it (serialized by sn). Also it is supposed that there - * could not be half-incremented halves. - */ +void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot); +int scst_check_hq_cmd(struct scst_cmd *cmd); - typeof(tgt_dev->expected_sn) e; - - tgt_dev->expected_sn++; - e = tgt_dev->expected_sn; - smp_mb(); /* write must be before def_cmd_count read */ - TRACE(TRACE_DEBUG/*TRACE_SCSI_SERIALIZING*/, "Next expected_sn: %d", e); - return e; -} - -void scst_inc_expected_sn_unblock(struct scst_tgt_dev *tgt_dev, +void scst_unblock_deferred(struct scst_tgt_dev *tgt_dev, struct scst_cmd *cmd_sn); int scst_cmd_thread(void *arg); @@ -386,6 +382,16 @@ static inline int scst_is_ua_command(struct scst_cmd *cmd) (cmd->cdb[0] != REPORT_LUNS)); } +static inline int scst_is_implicit_hq(struct scst_cmd *cmd) +{ + return ((cmd->cdb[0] == INQUIRY) || + (cmd->cdb[0] == REPORT_LUNS) || + ((cmd->dev->type == TYPE_DISK) && + ((cmd->cdb[0] == READ_CAPACITY) || + ((cmd->cdb[0] == SERVICE_ACTION_IN) && + ((cmd->cdb[1] & 0x1f) == SAI_READ_CAPACITY_16))))); +} + /* * Returns 1, if cmd's CDB is locally handled by SCST and 0 otherwise. * Dev handlers parse() and dev_done() not called for such commands. @@ -441,18 +447,28 @@ static inline void scst_unblock_dev(struct scst_device *dev) spin_unlock_bh(&dev->dev_lock); } -static inline void scst_dec_on_dev_cmd(struct scst_cmd *cmd) +static inline void __scst_dec_on_dev_cmd(struct scst_device *dev, + int cmd_blocking) { - if (cmd->blocking) { + if (cmd_blocking) + scst_unblock_dev(dev); + atomic_dec(&dev->on_dev_count); + smp_mb__after_atomic_dec(); + if (unlikely(dev->block_count != 0)) + wake_up_all(&dev->on_dev_waitQ); +} + +static inline int scst_dec_on_dev_cmd(struct scst_cmd *cmd, int defer) +{ + int cmd_blocking = cmd->blocking; + if (cmd_blocking) { TRACE_MGMT_DBG("cmd %p (tag %d): unblocking dev %p", cmd, cmd->tag, cmd->dev); cmd->blocking = 0; - scst_unblock_dev(cmd->dev); } - atomic_dec(&cmd->dev->on_dev_count); - smp_mb__after_atomic_dec(); - if (unlikely(cmd->dev->block_count != 0)) - wake_up_all(&cmd->dev->on_dev_waitQ); + if (!defer) + __scst_dec_on_dev_cmd(cmd->dev, cmd_blocking); + return cmd_blocking; } static inline void __scst_get(int barrier) @@ -551,4 +567,10 @@ static inline int tm_dbg_is_release(void) } #endif /* DEBUG_TM */ +#ifdef DEBUG_SN +void scst_check_debug_sn(struct scst_cmd *cmd); +#else +static inline void scst_check_debug_sn(struct scst_cmd *cmd) {} +#endif + #endif /* __SCST_PRIV_H */ diff --git a/scst/src/scst_proc.c b/scst/src/scst_proc.c index f5038b460..66fd9c823 100644 --- a/scst/src/scst_proc.c +++ b/scst/src/scst_proc.c @@ -1606,6 +1606,10 @@ static int scst_version_info_show(struct seq_file *seq, void *v) seq_printf(seq, "Strict serializing enabled\n"); #endif +#ifdef SINGLE_DEFAULT_GROUP + seq_printf(seq, "Single default group\n"); +#endif + #ifdef EXTRACHECKS seq_printf(seq, "EXTRACHECKS\n"); #endif @@ -1630,6 +1634,10 @@ static int scst_version_info_show(struct seq_file *seq, void *v) seq_printf(seq, "DEBUG_OOM\n"); #endif +#ifdef DEBUG_SN + seq_printf(seq, "DEBUG_SN\n"); +#endif + TRACE_EXIT(); return 0; } diff --git a/scst/src/scst_targ.c b/scst/src/scst_targ.c index 661d9a311..d36a1ae70 100644 --- a/scst/src/scst_targ.c +++ b/scst/src/scst_targ.c @@ -30,21 +30,10 @@ #include "scsi_tgt.h" #include "scst_priv.h" +static void scst_cmd_set_sn(struct scst_cmd *cmd); static int __scst_init_cmd(struct scst_cmd *cmd); static int scst_process_active_cmd(struct scst_cmd *cmd, int context); -/* No locks */ -static inline void scst_cmd_set_sn(struct scst_cmd *cmd) -{ - /* ToDo: cmd->queue_type */ - - cmd->sn = atomic_inc_return(&cmd->tgt_dev->curr_sn); - cmd->no_sn = 0; - - TRACE(TRACE_DEBUG/*TRACE_SCSI_SERIALIZING*/, "cmd(%p)->sn: %d", - cmd, cmd->sn); -} - static inline void scst_schedule_tasklet(struct scst_cmd *cmd) { struct scst_tasklet *t = &scst_tasklets[smp_processor_id()]; @@ -259,8 +248,12 @@ active: case SCST_CONTEXT_THREAD: spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags); TRACE_DBG("Adding cmd %p to active cmd list", cmd); - list_add_tail(&cmd->cmd_list_entry, - &cmd->cmd_lists->active_cmd_list); + if (unlikely(cmd->head_of_queue)) + list_add(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + else + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); wake_up(&cmd->cmd_lists->cmd_list_waitQ); spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags); break; @@ -275,7 +268,6 @@ static int scst_parse_cmd(struct scst_cmd *cmd) { int res = SCST_CMD_STATE_RES_CONT_SAME; int state; - struct scst_tgt_dev *tgt_dev_saved = cmd->tgt_dev; struct scst_device *dev = cmd->dev; struct scst_info_cdb cdb_info; int atomic = scst_cmd_atomic(cmd); @@ -446,13 +438,6 @@ static int scst_parse_cmd(struct scst_cmd *cmd) set_dir = 0; break; - case SCST_CMD_STATE_REINIT: - cmd->tgt_dev_saved = tgt_dev_saved; - cmd->state = state; - res = SCST_CMD_STATE_RES_RESTART; - set_dir = 0; - break; - default: if (state >= 0) { PRINT_ERROR_PR("Dev handler %s parse() returned " @@ -912,7 +897,12 @@ void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context, scst_check_retries(cmd->tgt); spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags); TRACE_DBG("Adding cmd %p to active cmd list", cmd); - list_add_tail(&cmd->cmd_list_entry, &cmd->cmd_lists->active_cmd_list); + if (unlikely(cmd->head_of_queue)) + list_add(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + else + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); wake_up(&cmd->cmd_lists->cmd_list_waitQ); spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags); break; @@ -1165,7 +1155,7 @@ static void scst_do_cmd_done(struct scst_cmd *cmd, int result, cmd->completed = 1; - scst_dec_on_dev_cmd(cmd); + scst_dec_on_dev_cmd(cmd, 0); type = cmd->dev->handler->type; if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) && @@ -1301,7 +1291,7 @@ static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state) sBUG_ON(in_irq()); - scst_dec_on_dev_cmd(cmd); + scst_dec_on_dev_cmd(cmd, 0); if (next_state == SCST_CMD_STATE_DEFAULT) next_state = SCST_CMD_STATE_DEV_DONE; @@ -1408,8 +1398,8 @@ static int scst_report_luns_local(struct scst_cmd *cmd) buffer_size); goto out_put_hw_err; } - buffer[offs] = (tgt_dev->acg_dev->lun >> 8) & 0xff; - buffer[offs+1] = tgt_dev->acg_dev->lun & 0xff; + buffer[offs] = (tgt_dev->lun >> 8) & 0xff; + buffer[offs+1] = tgt_dev->lun & 0xff; offs += 8; } inc_dev_cnt: @@ -1735,6 +1725,15 @@ static int scst_do_send_to_midlev(struct scst_cmd *cmd) TRACE_ENTRY(); + /* Check here to let an out of SN cmd be queued w/o context switch */ + if (scst_cmd_atomic(cmd) && !cmd->dev->handler->exec_atomic) { + TRACE_DBG("Dev handler %s exec() can not be " + "called in atomic context, rescheduling to the thread", + cmd->dev->handler->name); + rc = SCST_EXEC_NEED_THREAD; + goto out; + } + cmd->sent_to_midlev = 1; cmd->state = SCST_CMD_STATE_EXECUTING; cmd->scst_cmd_done = scst_cmd_done_local; @@ -1870,6 +1869,48 @@ out_aborted: goto out; } +/* No locks */ +void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot) +{ + if (slot == NULL) + goto inc; + + /* Optimized for lockless fast path */ + + TRACE_SN("Slot %d, *cur_sn_slot %d", slot-tgt_dev->sn_slots, + atomic_read(slot)); + + if (!atomic_dec_and_test(slot)) + goto out; + + TRACE_SN("Slot is 0 (num_free_sn_slots=%d)", + tgt_dev->num_free_sn_slots); + if (tgt_dev->num_free_sn_slots != ARRAY_SIZE(tgt_dev->sn_slots)) { + spin_lock_irq(&tgt_dev->sn_lock); + if (tgt_dev->num_free_sn_slots != ARRAY_SIZE(tgt_dev->sn_slots)) { + tgt_dev->num_free_sn_slots++; + TRACE_SN("Incremented num_free_sn_slots (%d)", + tgt_dev->num_free_sn_slots); + if (tgt_dev->num_free_sn_slots == 0) + tgt_dev->cur_sn_slot = slot; + } + spin_unlock_irq(&tgt_dev->sn_lock); + } + +inc: + /* + * No locks is needed, because only one thread at time can + * be here (serialized by sn). Also it is supposed that there + * could not be half-incremented halves. + */ + tgt_dev->expected_sn++; + smp_mb(); /* write must be before def_cmd_count read */ + TRACE_SN("Next expected_sn: %d", tgt_dev->expected_sn); + +out: + return; +} + static int scst_send_to_midlev(struct scst_cmd *cmd) { int res, rc; @@ -1877,20 +1918,11 @@ static int scst_send_to_midlev(struct scst_cmd *cmd) struct scst_device *dev = cmd->dev; int expected_sn; int count; - int atomic = scst_cmd_atomic(cmd); TRACE_ENTRY(); res = SCST_CMD_STATE_RES_CONT_NEXT; - if (atomic && !dev->handler->exec_atomic) { - TRACE_DBG("Dev handler %s exec() can not be " - "called in atomic context, rescheduling to the thread", - dev->handler->name); - res = SCST_CMD_STATE_RES_NEED_THREAD; - goto out; - } - if (unlikely(scst_inc_on_dev_cmd(cmd) != 0)) goto out; @@ -1903,7 +1935,7 @@ static int scst_send_to_midlev(struct scst_cmd *cmd) TRACE_DBG("%s", "scst_do_send_to_midlev() requested " "thread context, rescheduling"); res = SCST_CMD_STATE_RES_NEED_THREAD; - scst_dec_on_dev_cmd(cmd); + scst_dec_on_dev_cmd(cmd, 0); goto out_dec_cmd_count; } else { sBUG_ON(rc != SCST_EXEC_COMPLETED); @@ -1913,15 +1945,35 @@ static int scst_send_to_midlev(struct scst_cmd *cmd) EXTRACHECKS_BUG_ON(cmd->no_sn); + if (unlikely(cmd->head_of_queue)) { + /* + * W/o get() there will be a race, when cmd is executed and + * destroyed before "goto out_unplug" + */ + scst_cmd_get(cmd); + if (scst_check_hq_cmd(cmd)) { + scst_cmd_put(cmd); + goto exec; + } else { + scst_dec_on_dev_cmd(cmd, 0); + scst_cmd_put(cmd); + goto out_unplug; + } + } + expected_sn = tgt_dev->expected_sn; - if (cmd->sn != expected_sn) { - spin_lock_bh(&tgt_dev->sn_lock); + /* Optimized for lockless fast path */ + if ((cmd->sn != expected_sn) || unlikely(test_bit(SCST_TGT_DEV_HQ_ACTIVE, + &tgt_dev->tgt_dev_flags))) { + spin_lock_irq(&tgt_dev->sn_lock); tgt_dev->def_cmd_count++; smp_mb(); - barrier(); /* to reread expected_sn */ + barrier(); /* to reread expected_sn & hq_cmd_active */ expected_sn = tgt_dev->expected_sn; - if (cmd->sn != expected_sn) { - scst_dec_on_dev_cmd(cmd); + if ((cmd->sn != expected_sn) || test_bit(SCST_TGT_DEV_HQ_ACTIVE, + &tgt_dev->tgt_dev_flags)) { + /* We are under IRQ lock, but dev->dev_lock is BH one */ + int cmd_blocking = scst_dec_on_dev_cmd(cmd, 1); if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) { /* Necessary to allow aborting out of sn cmds */ TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %d)", @@ -1930,31 +1982,46 @@ static int scst_send_to_midlev(struct scst_cmd *cmd) cmd->state = SCST_CMD_STATE_DEV_DONE; res = SCST_CMD_STATE_RES_CONT_SAME; } else { - TRACE(TRACE_SCSI_SERIALIZING, "Delaying cmd %p " - "(sn=%d, expected_sn=%d)", cmd, - cmd->sn, expected_sn); + TRACE_SN("Deferring cmd %p (sn=%d, " + "expected_sn=%d, hq_cmd_active=%d)", cmd, + cmd->sn, expected_sn, + test_bit(SCST_TGT_DEV_HQ_ACTIVE, + &tgt_dev->tgt_dev_flags)); list_add_tail(&cmd->sn_cmd_list_entry, &tgt_dev->deferred_cmd_list); } - spin_unlock_bh(&tgt_dev->sn_lock); + spin_unlock_irq(&tgt_dev->sn_lock); /* !! At this point cmd can be already freed !! */ + __scst_dec_on_dev_cmd(dev, cmd_blocking); goto out_dec_cmd_count; } else { - TRACE(TRACE_SCSI_SERIALIZING, "Somebody incremented " - "expected_sn %d, continuing", expected_sn); + TRACE_SN("Somebody incremented expected_sn %d, " + "continuing", expected_sn); tgt_dev->def_cmd_count--; - spin_unlock_bh(&tgt_dev->sn_lock); + spin_unlock_irq(&tgt_dev->sn_lock); } } +exec: count = 0; while(1) { + atomic_t *slot = cmd->sn_slot; + int hq = cmd->head_of_queue; rc = scst_do_send_to_midlev(cmd); if (rc == SCST_EXEC_NEED_THREAD) { TRACE_DBG("%s", "scst_do_send_to_midlev() requested " "thread context, rescheduling"); res = SCST_CMD_STATE_RES_NEED_THREAD; - scst_dec_on_dev_cmd(cmd); + if (unlikely(hq)) { + TRACE_SN("Rescheduling HQ cmd %p", cmd); + spin_lock_irq(&tgt_dev->sn_lock); + clear_bit(SCST_TGT_DEV_HQ_ACTIVE, + &tgt_dev->tgt_dev_flags); + list_add(&cmd->sn_cmd_list_entry, + &tgt_dev->hq_cmd_list); + spin_unlock_irq(&tgt_dev->sn_lock); + } + scst_dec_on_dev_cmd(cmd, 0); if (count != 0) goto out_unplug; else @@ -1963,8 +2030,9 @@ static int scst_send_to_midlev(struct scst_cmd *cmd) sBUG_ON(rc != SCST_EXEC_COMPLETED); /* !! At this point cmd can be already freed !! */ count++; - expected_sn = __scst_inc_expected_sn(tgt_dev); - cmd = scst_check_deferred_commands(tgt_dev, expected_sn); + if (likely(!hq)) + scst_inc_expected_sn(tgt_dev, slot); + cmd = scst_check_deferred_commands(tgt_dev); if (cmd == NULL) break; if (unlikely(scst_inc_on_dev_cmd(cmd) != 0)) @@ -2145,11 +2213,6 @@ static int scst_dev_done(struct scst_cmd *cmd) } switch (state) { - case SCST_CMD_STATE_REINIT: - cmd->state = state; - res = SCST_CMD_STATE_RES_RESTART; - break; - case SCST_CMD_STATE_DEV_PARSE: case SCST_CMD_STATE_PREPARE_SPACE: case SCST_CMD_STATE_RDY_TO_XFER: @@ -2197,16 +2260,14 @@ static int scst_xmit_response(struct scst_cmd *cmd) TRACE_ENTRY(); - /* + /* * Check here also in order to avoid unnecessary delays of other * commands. */ - if (unlikely(cmd->sent_to_midlev == 0) && - (cmd->tgt_dev != NULL)) - { - TRACE(TRACE_SCSI_SERIALIZING, - "cmd %p was not sent to mid-lev (sn %d)", cmd, cmd->sn); - scst_inc_expected_sn_unblock(cmd->tgt_dev, cmd); + if (unlikely(!cmd->sent_to_midlev) && (cmd->tgt_dev != NULL)) { + TRACE_SN("cmd %p was not sent to mid-lev (sn %d)", + cmd, cmd->sn); + scst_unblock_deferred(cmd->tgt_dev, cmd); cmd->sent_to_midlev = 1; } @@ -2367,6 +2428,97 @@ static int scst_finish_cmd(struct scst_cmd *cmd) return res; } +/* + * No locks, but it must be externally serialized (see comment for + * scst_cmd_init_done() in scsi_tgt.h) + */ +static void scst_cmd_set_sn(struct scst_cmd *cmd) +{ + struct scst_tgt_dev *tgt_dev = cmd->tgt_dev; + unsigned long flags; + + if (scst_is_implicit_hq(cmd)) { + TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd); + cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE; + } + + /* Optimized for lockless fast path */ + + scst_check_debug_sn(cmd); + + switch(cmd->queue_type) { + case SCST_CMD_QUEUE_SIMPLE: + case SCST_CMD_QUEUE_UNTAGGED: + if (likely(tgt_dev->num_free_sn_slots >= 0)) { + if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) { + tgt_dev->curr_sn++; + TRACE_SN("Incremented curr_sn %d", + tgt_dev->curr_sn); + } + cmd->sn_slot = tgt_dev->cur_sn_slot; + cmd->sn = tgt_dev->curr_sn; + tgt_dev->prev_cmd_ordered = 0; + } else { + TRACE(TRACE_MINOR, "%s", "Not enough SN slots"); + goto ordered; + } + break; + + default: + PRINT_ERROR_PR("Unsupported queue type %d, treating it as " + "ORDERED", cmd->queue_type); + cmd->queue_type = SCST_CMD_QUEUE_ORDERED; + /* go through */ + case SCST_CMD_QUEUE_ORDERED: + TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p " + "(op %x)", cmd, cmd->cdb[0]); + if (!tgt_dev->prev_cmd_ordered) { + spin_lock_irqsave(&tgt_dev->sn_lock, flags); + tgt_dev->num_free_sn_slots--; + smp_mb(); + if ((tgt_dev->num_free_sn_slots >= 0) && + (atomic_read(tgt_dev->cur_sn_slot) > 0)) { + do { + tgt_dev->cur_sn_slot++; + if (tgt_dev->cur_sn_slot == + tgt_dev->sn_slots + + ARRAY_SIZE(tgt_dev->sn_slots)) + tgt_dev->cur_sn_slot = tgt_dev->sn_slots; + } while(atomic_read(tgt_dev->cur_sn_slot) != 0); + TRACE_SN("New cur SN slot %d", + tgt_dev->cur_sn_slot-tgt_dev->sn_slots); + } else + tgt_dev->num_free_sn_slots++; + spin_unlock_irqrestore(&tgt_dev->sn_lock, flags); + } +ordered: + tgt_dev->prev_cmd_ordered = 1; + tgt_dev->curr_sn++; + cmd->sn = tgt_dev->curr_sn; + break; + + case SCST_CMD_QUEUE_HEAD_OF_QUEUE: + TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p " + "(op %x)", cmd, cmd->cdb[0]); + cmd->head_of_queue = 1; + spin_lock_irqsave(&tgt_dev->sn_lock, flags); + /* Add in the head as required by SAM */ + list_add(&cmd->sn_cmd_list_entry, &tgt_dev->hq_cmd_list); + spin_unlock_irqrestore(&tgt_dev->sn_lock, flags); + break; + } + + TRACE_SN("cmd(%p)->sn: %d (tgt_dev %p, *cur_sn_slot %d, " + "num_free_sn_slots %d, prev_cmd_ordered %d, " + "cur_sn_slot %d)", cmd, cmd->sn, tgt_dev, + atomic_read(tgt_dev->cur_sn_slot), + tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered, + tgt_dev->cur_sn_slot-tgt_dev->sn_slots); + + cmd->no_sn = 0; + return; +} + /* * Returns 0 on success, > 0 when we need to wait for unblock, * < 0 if there is no device (lun) or device type handler. @@ -2390,28 +2542,19 @@ static int scst_translate_lun(struct scst_cmd *cmd) list_for_each_entry(tgt_dev, &cmd->sess->sess_tgt_dev_list, sess_tgt_dev_list_entry) { - if (tgt_dev->acg_dev->lun == cmd->lun) { + if (tgt_dev->lun == cmd->lun) { TRACE_DBG("tgt_dev %p found", tgt_dev); - if (unlikely(tgt_dev->acg_dev->dev->handler == NULL)) { + if (unlikely(tgt_dev->dev->handler == NULL)) { PRINT_INFO_PR("Dev handler for device " "%Ld is NULL, the device will not be " "visible remotely", (uint64_t)cmd->lun); break; } - if (cmd->state == SCST_CMD_STATE_REINIT) { - atomic_dec(&cmd->tgt_dev_saved->cmd_count); - TRACE(TRACE_SCSI_SERIALIZING, - "SCST_CMD_STATE_REINIT: " - "incrementing expected_sn on tgt_dev_saved %p", - cmd->tgt_dev_saved); - scst_inc_expected_sn_unblock( - cmd->tgt_dev_saved, cmd); - } cmd->cmd_lists = tgt_dev->p_cmd_lists; cmd->tgt_dev = tgt_dev; - cmd->dev = tgt_dev->acg_dev->dev; + cmd->dev = tgt_dev->dev; res = 0; break; @@ -2521,8 +2664,12 @@ restart: spin_lock(&cmd->cmd_lists->cmd_list_lock); TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd); - list_add_tail(&cmd->cmd_list_entry, - &cmd->cmd_lists->active_cmd_list); + if (unlikely(cmd->head_of_queue)) + list_add(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); + else + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_lists->active_cmd_list); wake_up(&cmd->cmd_lists->cmd_list_waitQ); spin_unlock(&cmd->cmd_lists->cmd_list_lock); @@ -2596,7 +2743,6 @@ static int scst_process_active_cmd(struct scst_cmd *cmd, int context) cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC); -restart: do { switch (cmd->state) { case SCST_CMD_STATE_DEV_PARSE: @@ -2677,23 +2823,6 @@ restart: } wake_up(&cmd->cmd_lists->cmd_list_waitQ); spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock); - } else if (res == SCST_CMD_STATE_RES_RESTART) { - if (cmd->state == SCST_CMD_STATE_REINIT) { - int rc; - rc = scst_init_cmd(cmd, context); - if (rc == context) - goto restart; - else if (rc == SCST_CONTEXT_THREAD) { - spin_lock_irq(&cmd->cmd_lists->cmd_list_lock); - TRACE_DBG("Adding cmd %p to active cmd list", - cmd); - list_add_tail(&cmd->cmd_list_entry, - &cmd->cmd_lists->active_cmd_list); - spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock); - } else if (rc >= 0) - sBUG(); - } else - sBUG(); } else sBUG(); @@ -2721,8 +2850,7 @@ static void scst_do_job_active(struct list_head *cmd_list, spin_unlock_irq(cmd_list_lock); rc = scst_process_active_cmd(cmd, context); if (unlikely((rc != SCST_CMD_STATE_RES_CONT_NEXT) && - (rc != SCST_CMD_STATE_RES_NEED_THREAD) && - (rc != SCST_CMD_STATE_RES_RESTART))) { + (rc != SCST_CMD_STATE_RES_NEED_THREAD))) { sBUG(); } spin_lock_irq(cmd_list_lock); @@ -2780,14 +2908,17 @@ int scst_cmd_thread(void *arg) } spin_unlock_irq(&p_cmd_lists->cmd_list_lock); +#ifdef EXTRACHECKS /* * If kthread_should_stop() is true, we are guaranteed to be either * on the module unload, or there must be at least one other thread to * process the commands lists. */ -//!!ToDo sBUG_ON((scst_threads_info.nr_cmd_threads == 1) && -// (!list_empty(&scst_cmd_lists) || -// !list_empty(&scst_active_cmd_lists))); + if (p_cmd_lists == &scst_main_cmd_lists) { + sBUG_ON((scst_threads_info.nr_cmd_threads == 1) && + !list_empty(&scst_main_cmd_lists.active_cmd_list)); + } +#endif TRACE_EXIT(); return 0; @@ -2836,7 +2967,7 @@ static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd) list_for_each_entry(tgt_dev, &mcmd->sess->sess_tgt_dev_list, sess_tgt_dev_list_entry) { - if (tgt_dev->acg_dev->lun == mcmd->lun) { + if (tgt_dev->lun == mcmd->lun) { TRACE_DBG("tgt_dev %p found", tgt_dev); mcmd->mcmd_tgt_dev = tgt_dev; res = 0; @@ -2900,7 +3031,7 @@ static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd, struct scst_tgt_dev *tgt_dev, int set_status) { int res = SCST_DEV_TM_NOT_COMPLETED; - struct scst_dev_type *h = tgt_dev->acg_dev->dev->handler; + struct scst_dev_type *h = tgt_dev->dev->handler; if (h->task_mgmt_fn) { TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)", @@ -3066,7 +3197,7 @@ static void scst_unblock_aborted_cmds(int scst_mutex_held) local_irq_disable(); list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, dev_tgt_dev_list_entry) { - spin_lock_bh(&tgt_dev->sn_lock); + spin_lock(&tgt_dev->sn_lock); list_for_each_entry_safe(cmd, tcmd, &tgt_dev->deferred_cmd_list, sn_cmd_list_entry) { @@ -3077,7 +3208,7 @@ static void scst_unblock_aborted_cmds(int scst_mutex_held) list_del(&cmd->sn_cmd_list_entry); } } - spin_unlock_bh(&tgt_dev->sn_lock); + spin_unlock(&tgt_dev->sn_lock); } local_irq_enable(); } @@ -3105,7 +3236,7 @@ static void __scst_abort_task_set(struct scst_mgmt_cmd *mcmd, search_cmd_list_entry) { if ((cmd->tgt_dev == tgt_dev) || ((cmd->tgt_dev == NULL) && - (cmd->lun == tgt_dev->acg_dev->lun))) + (cmd->lun == tgt_dev->lun))) scst_abort_cmd(cmd, mcmd, other_ini, 0); } spin_unlock_irq(&sess->sess_list_lock); @@ -3121,10 +3252,10 @@ static int scst_abort_task_set(struct scst_mgmt_cmd *mcmd) { int res; struct scst_tgt_dev *tgt_dev = mcmd->mcmd_tgt_dev; - struct scst_device *dev = tgt_dev->acg_dev->dev; + struct scst_device *dev = tgt_dev->dev; TRACE(TRACE_MGMT, "Aborting task set (lun=%d, mcmd=%p)", - tgt_dev->acg_dev->lun, mcmd); + tgt_dev->lun, mcmd); spin_lock_bh(&dev->dev_lock); __scst_block_dev(dev); @@ -3305,12 +3436,11 @@ static int scst_lun_reset(struct scst_mgmt_cmd *mcmd) { int res, rc; struct scst_tgt_dev *tgt_dev = mcmd->mcmd_tgt_dev; - struct scst_device *dev = tgt_dev->acg_dev->dev; + struct scst_device *dev = tgt_dev->dev; TRACE_ENTRY(); - TRACE(TRACE_MGMT, "Resetting lun %d (mcmd %p)", tgt_dev->acg_dev->lun, - mcmd); + TRACE(TRACE_MGMT, "Resetting lun %d (mcmd %p)", tgt_dev->lun, mcmd); spin_lock_bh(&dev->dev_lock); __scst_block_dev(dev); @@ -3360,7 +3490,7 @@ static int scst_abort_all_nexus_loss_sess(struct scst_mgmt_cmd *mcmd, list_for_each_entry(tgt_dev, &sess->sess_tgt_dev_list, sess_tgt_dev_list_entry) { - struct scst_device *dev = tgt_dev->acg_dev->dev; + struct scst_device *dev = tgt_dev->dev; int rc; spin_lock_bh(&dev->dev_lock); @@ -3533,7 +3663,7 @@ static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd) case SCST_ABORT_TASK_SET: case SCST_CLEAR_TASK_SET: case SCST_LUN_RESET: - scst_unblock_dev(mcmd->mcmd_tgt_dev->acg_dev->dev); + scst_unblock_dev(mcmd->mcmd_tgt_dev->dev); break; case SCST_TARGET_RESET: @@ -3551,7 +3681,7 @@ static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd) down(&scst_mutex); list_for_each_entry(tgt_dev, &mcmd->sess->sess_tgt_dev_list, sess_tgt_dev_list_entry) { - scst_unblock_dev(tgt_dev->acg_dev->dev); + scst_unblock_dev(tgt_dev->dev); } up(&scst_mutex); break;