From d0928114c02536a0ecdb19f24c915ca494866c29 Mon Sep 17 00:00:00 2001 From: Vladislav Bolkhovitin Date: Wed, 13 Apr 2011 17:18:27 +0000 Subject: [PATCH] Commands ordering/serialization improvements git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@3382 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- scst/include/scst.h | 108 +++++++----- scst/include/scst_const.h | 8 +- scst/src/scst_lib.c | 341 ++++++++++++++++++++------------------ scst/src/scst_main.c | 10 +- scst/src/scst_pres.c | 6 +- scst/src/scst_pres.h | 82 ++++++--- scst/src/scst_priv.h | 55 +++--- scst/src/scst_targ.c | 252 ++++++++++++++++++---------- 8 files changed, 510 insertions(+), 352 deletions(-) diff --git a/scst/include/scst.h b/scst/include/scst.h index bbdec2023..52f75af70 100644 --- a/scst/include/scst.h +++ b/scst/include/scst.h @@ -1691,6 +1691,34 @@ struct scst_cmd_threads { struct list_head lists_list_entry; }; +/* + * Used to execute cmd's in order of arrival, honoring SCSI task attributes + */ +struct scst_order_data { + /* + * Protected by sn_lock, except expected_sn, which is protected by + * itself. Curr_sn must have the same size as expected_sn to + * overflow simultaneously. + */ + int def_cmd_count; + spinlock_t sn_lock; + unsigned int expected_sn; + unsigned int curr_sn; + int hq_cmd_count; + struct list_head deferred_cmd_list; + struct list_head skipped_sn_list; + + /* + * Set if the prev cmd was ORDERED. Size and, hence, alignment must + * allow unprotected modifications independently to the neighbour fields. + */ + unsigned long prev_cmd_ordered; + + int num_free_sn_slots; /* if it's <0, then all slots are busy */ + atomic_t *cur_sn_slot; + atomic_t sn_slots[15]; +}; + /* * SCST command, analog of I_T_L_Q nexus or task */ @@ -1744,6 +1772,12 @@ struct scst_cmd { /* Set if the device was blocked by scst_check_blocked_dev() */ unsigned int unblock_dev:1; + /* Set if this cmd incremented dev->pr_readers_count */ + unsigned int dec_pr_readers_count_needed:1; + + /* Set if scst_dec_on_dev_cmd() call is needed on the cmd's finish */ + unsigned int dec_on_dev_needed:1; + /* Set if cmd is queued as hw pending */ unsigned int cmd_hw_pending:1; @@ -1851,7 +1885,10 @@ struct scst_cmd { struct scst_tgt *tgt; /* to save extra dereferences */ struct scst_device *dev; /* to save extra dereferences */ - struct scst_tgt_dev *tgt_dev; /* corresponding device for this cmd */ + /* corresponding I_T_L device for this cmd */ + struct scst_tgt_dev *tgt_dev; + + struct scst_order_data *cur_order_data; /* to save extra dereferences */ uint64_t lun; /* LUN for this cmd */ @@ -2148,6 +2185,9 @@ struct scst_device { /* If set, dev is read only */ unsigned short rd_only:1; + /* Set, if a strictly serialized cmd is waiting blocked */ + unsigned short strictly_serialized_cmd_waiting:1; + /**************************************************************/ /************************************************************* @@ -2171,14 +2211,28 @@ struct scst_device { /**************************************************************/ + /* How many cmds alive on this dev */ + atomic_t dev_cmd_count; + + spinlock_t dev_lock; /* device lock */ + /* * How many times device was blocked for new cmds execution. - * Protected by dev_lock + * Protected by dev_lock. */ int block_count; - /* How many cmds alive on this dev */ - atomic_t dev_cmd_count; + /* + * How many there are "on_dev" commands, i.e. ones who passed + * scst_check_blocked_dev(). Protected by dev_lock. + */ + int on_dev_cmd_count; + + /* + * How many threads are checking commands for PR allowance. + * Protected by dev_lock. + */ + int pr_readers_count; /* * Set if dev is persistently reserved. Protected by dev_pr_mutex. @@ -2192,12 +2246,6 @@ struct scst_device { */ unsigned int pr_writer_active:1; - /* - * How many threads are checking commands for PR allowance. Used to - * implement lockless read-only fast path. - */ - atomic_t pr_readers_count; - struct scst_dev_type *handler; /* corresponding dev handler */ /* Used for storage of dev handler private stuff */ @@ -2247,27 +2295,26 @@ struct scst_device { */ int not_pr_supporting_tgt_devs_num; + struct scst_order_data dev_order_data; + /* Persist through power loss files */ char *pr_file_name; char *pr_file_name1; /**************************************************************/ - spinlock_t dev_lock; /* device lock */ - - struct list_head blocked_cmd_list; /* protected by dev_lock */ + /* List of blocked commands, protected by dev_lock. */ + struct list_head blocked_cmd_list; /* A list entry used during TM, protected by scst_mutex */ struct list_head tm_dev_list_entry; - /* Virtual device internal ID */ - int virt_id; + int virt_id; /* virtual device internal ID */ /* Pointer to virtual device name, for convenience only */ char *virt_name; - /* List entry in global devices list */ - struct list_head dev_list_entry; + struct list_head dev_list_entry; /* list entry in global devices list */ /* * List of tgt_dev's, one per session, protected by scst_mutex or @@ -2346,31 +2393,8 @@ struct scst_tgt_dev { /* How many cmds alive on this dev in this session */ atomic_t tgt_dev_cmd_count; - /* - * Used to execute cmd's in order of arrival, honoring SCSI task - * attributes. - * - * Protected by sn_lock, except expected_sn, which is protected by - * itself. Curr_sn must have the same size as expected_sn to - * overflow simultaneously. - */ - int def_cmd_count; - spinlock_t sn_lock; - unsigned int expected_sn; - unsigned int curr_sn; - int hq_cmd_count; - struct list_head deferred_cmd_list; - struct list_head skipped_sn_list; - - /* - * Set if the prev cmd was ORDERED. Size and, hence, alignment must - * allow unprotected modifications independently to the neighbour fields. - */ - unsigned long prev_cmd_ordered; - - int num_free_sn_slots; /* if it's <0, then all slots are busy */ - atomic_t *cur_sn_slot; - atomic_t sn_slots[15]; + struct scst_order_data *curr_order_data; + struct scst_order_data tgt_dev_order_data; /* List of scst_thr_data_hdr and lock */ spinlock_t thr_data_lock; diff --git a/scst/include/scst_const.h b/scst/include/scst_const.h index 50148e9c7..aba3d3f5b 100644 --- a/scst/include/scst_const.h +++ b/scst/include/scst_const.h @@ -175,10 +175,6 @@ enum scst_cmd_queue_type { /************************************************************* ** CDB flags - ** - ** Implicit ordered used for commands which need calm environment - ** without any simultaneous activities. For instance, for MODE - ** SELECT it is needed to correctly generate its UA. *************************************************************/ enum scst_cdb_flags { SCST_TRANSFER_LEN_TYPE_FIXED = 0x0001, @@ -196,8 +192,10 @@ enum scst_cdb_flags { SCST_WRITE_EXCL_ALLOWED = 0x1000, SCST_EXCL_ACCESS_ALLOWED = 0x2000, #ifdef CONFIG_SCST_TEST_IO_IN_SIRQ - SCST_TEST_IO_IN_SIRQ_ALLOWED = 0x8000, + SCST_TEST_IO_IN_SIRQ_ALLOWED = 0x4000, #endif + SCST_SERIALIZED = 0x8000, + SCST_STRICTLY_SERIALIZED = 0x10000|SCST_SERIALIZED, }; /************************************************************* diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index 6a2aca917..3f1b11ec3 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -134,7 +134,7 @@ struct scst_sdbops { uint8_t direction; /* init --> target: SCST_DATA_WRITE * target --> init: SCST_DATA_READ */ - uint16_t flags; /* opcode -- various flags */ + uint32_t flags; /* opcode -- various flags */ uint8_t off; /* length offset in cdb */ int (*get_trans_len)(struct scst_cmd *cmd, uint8_t off); }; @@ -271,13 +271,13 @@ static const struct scst_sdbops scst_scsi_op_table[] = { SCST_WRITE_EXCL_ALLOWED, 2, get_trans_len_3}, {0x15, "OMOOOOOOOOOOOOOO", "MODE SELECT(6)", - SCST_DATA_WRITE, FLAG_NONE, 4, get_trans_len_1}, + SCST_DATA_WRITE, SCST_STRICTLY_SERIALIZED, 4, get_trans_len_1}, {0x16, "MMMMMMMMMMMMMMMM", "RESERVE", - SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD| + SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED| SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED, 0, get_trans_len_none}, {0x17, "MMMMMMMMMMMMMMMM", "RELEASE", - SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD| + SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED| SCST_REG_RESERVE_ALLOWED| SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED, 0, get_trans_len_none}, @@ -433,7 +433,7 @@ static const struct scst_sdbops scst_scsi_op_table[] = { {0x4B, " O ", "PAUSE/RESUME", SCST_DATA_NONE, FLAG_NONE, 0, get_trans_len_none}, {0x4C, "OOOOOOOOOOOOOOOO", "LOG SELECT", - SCST_DATA_WRITE, FLAG_NONE, 7, get_trans_len_2}, + SCST_DATA_WRITE, SCST_STRICTLY_SERIALIZED, 7, get_trans_len_2}, {0x4D, "OOOOOOOOOOOOOOOO", "LOG SENSE", SCST_DATA_READ, SCST_SMALL_TIMEOUT| SCST_REG_RESERVE_ALLOWED| @@ -459,13 +459,15 @@ static const struct scst_sdbops scst_scsi_op_table[] = { {0x54, " O ", "SEND OPC INFORMATION", SCST_DATA_WRITE, FLAG_NONE, 7, get_trans_len_2}, {0x55, "OOOOOOOOOOOOOOOO", "MODE SELECT(10)", - SCST_DATA_WRITE, FLAG_NONE, 7, get_trans_len_2}, + SCST_DATA_WRITE, SCST_STRICTLY_SERIALIZED, 7, get_trans_len_2}, {0x56, "OOOOOOOOOOOOOOOO", "RESERVE(10)", - SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD, + SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED| + SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED, 0, get_trans_len_none}, {0x57, "OOOOOOOOOOOOOOOO", "RELEASE(10)", - SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD| - SCST_REG_RESERVE_ALLOWED, + SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED| + SCST_REG_RESERVE_ALLOWED| + SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED, 0, get_trans_len_none}, {0x58, " O ", "REPAIR TRACK", SCST_DATA_NONE, SCST_WRITE_MEDIUM, 0, get_trans_len_none}, @@ -477,15 +479,15 @@ static const struct scst_sdbops scst_scsi_op_table[] = { SCST_DATA_READ, FLAG_NONE, 7, get_trans_len_2}, {0x5D, " O ", "SEND CUE SHEET", SCST_DATA_WRITE, FLAG_NONE, 6, get_trans_len_3}, - {0x5E, "OOOOO OOOO ", "PERSISTENT RESERV IN", + {0x5E, "OOOOO OOOO ", "PERSISTENT RESERVE IN", SCST_DATA_READ, SCST_SMALL_TIMEOUT| - SCST_LOCAL_CMD| + SCST_LOCAL_CMD|SCST_SERIALIZED| SCST_WRITE_EXCL_ALLOWED| SCST_EXCL_ACCESS_ALLOWED, 5, get_trans_len_4}, - {0x5F, "OOOOO OOOO ", "PERSISTENT RESERV OUT", + {0x5F, "OOOOO OOOO ", "PERSISTENT RESERVE OUT", SCST_DATA_WRITE, SCST_SMALL_TIMEOUT| - SCST_LOCAL_CMD| + SCST_LOCAL_CMD|SCST_SERIALIZED| SCST_WRITE_EXCL_ALLOWED| SCST_EXCL_ACCESS_ALLOWED, 5, get_trans_len_4}, @@ -688,7 +690,6 @@ static void scst_alloc_set_UA(struct scst_tgt_dev *tgt_dev, const uint8_t *sense, int sense_len, int flags); static void scst_free_all_UA(struct scst_tgt_dev *tgt_dev); static void scst_release_space(struct scst_cmd *cmd); -static void scst_unblock_cmds(struct scst_device *dev); static void scst_clear_reservation(struct scst_tgt_dev *tgt_dev); static int scst_alloc_add_tgt_dev(struct scst_session *sess, struct scst_acg_dev *acg_dev, struct scst_tgt_dev **out_tgt_dev); @@ -2564,6 +2565,21 @@ void scst_free_tgt(struct scst_tgt *tgt) return; } +static void scst_init_order_data(struct scst_order_data *order_data) +{ + int i; + spin_lock_init(&order_data->sn_lock); + INIT_LIST_HEAD(&order_data->deferred_cmd_list); + INIT_LIST_HEAD(&order_data->skipped_sn_list); + order_data->curr_sn = (typeof(order_data->curr_sn))(-300); + order_data->expected_sn = order_data->curr_sn + 1; + order_data->num_free_sn_slots = ARRAY_SIZE(order_data->sn_slots)-1; + order_data->cur_sn_slot = &order_data->sn_slots[0]; + for (i = 0; i < (int)ARRAY_SIZE(order_data->sn_slots); i++) + atomic_set(&order_data->sn_slots[i], 0); + return; +} + /* Called under scst_mutex and suspended activity */ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev) { @@ -2590,7 +2606,6 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev) dev->queue_alg = SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER; mutex_init(&dev->dev_pr_mutex); - atomic_set(&dev->pr_readers_count, 0); dev->pr_generation = 0; dev->pr_is_set = 0; dev->pr_holder = NULL; @@ -2598,6 +2613,8 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev) dev->pr_type = TYPE_UNSPECIFIED; INIT_LIST_HEAD(&dev->dev_registrants_list); + scst_init_order_data(&dev->dev_order_data); + scst_init_threads(&dev->dev_cmd_threads); *out_dev = dev; @@ -3279,7 +3296,7 @@ static int scst_alloc_add_tgt_dev(struct scst_session *sess, struct scst_tgt_dev *tgt_dev; struct scst_device *dev = acg_dev->dev; struct list_head *head; - int i, sl; + int sl; uint8_t sense_buffer[SCST_STANDARD_SENSE_LEN]; TRACE_ENTRY(); @@ -3325,15 +3342,12 @@ static int scst_alloc_add_tgt_dev(struct scst_session *sess, INIT_LIST_HEAD(&tgt_dev->UA_list); spin_lock_init(&tgt_dev->thr_data_lock); INIT_LIST_HEAD(&tgt_dev->thr_data_list); - spin_lock_init(&tgt_dev->sn_lock); - INIT_LIST_HEAD(&tgt_dev->deferred_cmd_list); - INIT_LIST_HEAD(&tgt_dev->skipped_sn_list); - tgt_dev->curr_sn = (typeof(tgt_dev->curr_sn))(-300); - tgt_dev->expected_sn = tgt_dev->curr_sn + 1; - tgt_dev->num_free_sn_slots = ARRAY_SIZE(tgt_dev->sn_slots)-1; - tgt_dev->cur_sn_slot = &tgt_dev->sn_slots[0]; - for (i = 0; i < (int)ARRAY_SIZE(tgt_dev->sn_slots); i++) - atomic_set(&tgt_dev->sn_slots[i], 0); + + scst_init_order_data(&tgt_dev->tgt_dev_order_data); + if (dev->tst == SCST_CONTR_MODE_SEP_TASK_SETS) + tgt_dev->curr_order_data = &tgt_dev->tgt_dev_order_data; + else + tgt_dev->curr_order_data = &dev->dev_order_data; if (dev->handler->parse_atomic && dev->handler->alloc_data_buf_atomic && @@ -4208,7 +4222,8 @@ void scst_free_cmd(struct scst_cmd *cmd) if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) TRACE_MGMT_DBG("Freeing aborted cmd %p", cmd); - sBUG_ON(cmd->unblock_dev); + EXTRACHECKS_BUG_ON(cmd->unblock_dev || cmd->dec_on_dev_needed || + cmd->dec_pr_readers_count_needed); /* * Target driver can already free sg buffer before calling @@ -4253,8 +4268,8 @@ void scst_free_cmd(struct scst_cmd *cmd) "%d, target %s, LUN %lld, sn %d, expected_sn %d)", cmd, cmd->cdb[0], cmd->tgtt->name, (long long unsigned int)cmd->lun, - cmd->sn, cmd->tgt_dev->expected_sn); - scst_unblock_deferred(cmd->tgt_dev, cmd); + cmd->sn, cmd->cur_order_data->expected_sn); + scst_unblock_deferred(cmd->cur_order_data, cmd); } #endif @@ -6327,7 +6342,7 @@ void __scst_dev_check_set_UA(struct scst_device *dev, { TRACE_ENTRY(); - TRACE_MGMT_DBG("Processing UA dev %p", dev); + TRACE_MGMT_DBG("Processing UA dev %s", dev->virt_name); /* Check for reset UA */ if (scst_analyze_sense(sense, sense_len, SCST_SENSE_ASC_VALID, @@ -6364,25 +6379,25 @@ static void scst_free_all_UA(struct scst_tgt_dev *tgt_dev) } /* No locks */ -struct scst_cmd *__scst_check_deferred_commands(struct scst_tgt_dev *tgt_dev) +struct scst_cmd *__scst_check_deferred_commands(struct scst_order_data *order_data) { struct scst_cmd *res = NULL, *cmd, *t; - typeof(tgt_dev->expected_sn) expected_sn = tgt_dev->expected_sn; + typeof(order_data->expected_sn) expected_sn = order_data->expected_sn; - spin_lock_irq(&tgt_dev->sn_lock); + spin_lock_irq(&order_data->sn_lock); - if (unlikely(tgt_dev->hq_cmd_count != 0)) + if (unlikely(order_data->hq_cmd_count != 0)) goto out_unlock; restart: - list_for_each_entry_safe(cmd, t, &tgt_dev->deferred_cmd_list, + list_for_each_entry_safe(cmd, t, &order_data->deferred_cmd_list, sn_cmd_list_entry) { EXTRACHECKS_BUG_ON(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE); if (cmd->sn == expected_sn) { TRACE_SN("Deferred command %p (sn %d, set %d) found", cmd, cmd->sn, cmd->sn_set); - tgt_dev->def_cmd_count--; + order_data->def_cmd_count--; list_del(&cmd->sn_cmd_list_entry); if (res == NULL) res = cmd; @@ -6400,7 +6415,7 @@ restart: if (res != NULL) goto out_unlock; - list_for_each_entry(cmd, &tgt_dev->skipped_sn_list, + list_for_each_entry(cmd, &order_data->skipped_sn_list, sn_cmd_list_entry) { EXTRACHECKS_BUG_ON(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE); @@ -6415,21 +6430,21 @@ restart: cmd, (long long unsigned int)cmd->tag, cmd->sn); - tgt_dev->def_cmd_count--; + order_data->def_cmd_count--; list_del(&cmd->sn_cmd_list_entry); - spin_unlock_irq(&tgt_dev->sn_lock); + spin_unlock_irq(&order_data->sn_lock); if (test_and_set_bit(SCST_CMD_CAN_BE_DESTROYED, &cmd->cmd_flags)) scst_destroy_put_cmd(cmd); - scst_inc_expected_sn(tgt_dev, slot); - expected_sn = tgt_dev->expected_sn; - spin_lock_irq(&tgt_dev->sn_lock); + scst_inc_expected_sn(order_data, slot); + expected_sn = order_data->expected_sn; + spin_lock_irq(&order_data->sn_lock); goto restart; } } out_unlock: - spin_unlock_irq(&tgt_dev->sn_lock); + spin_unlock_irq(&order_data->sn_lock); return res; } @@ -6565,26 +6580,55 @@ bool scst_del_thr_data(struct scst_tgt_dev *tgt_dev, struct task_struct *tsk) return res; } +static void __scst_unblock_deferred(struct scst_order_data *order_data, + struct scst_cmd *out_of_sn_cmd) +{ + EXTRACHECKS_BUG_ON(!out_of_sn_cmd->sn_set); + + if (out_of_sn_cmd->sn == order_data->expected_sn) { + scst_inc_expected_sn(order_data, out_of_sn_cmd->sn_slot); + scst_make_deferred_commands_active(order_data); + } else { + out_of_sn_cmd->out_of_sn = 1; + spin_lock_irq(&order_data->sn_lock); + order_data->def_cmd_count++; + list_add_tail(&out_of_sn_cmd->sn_cmd_list_entry, + &order_data->skipped_sn_list); + TRACE_SN("out_of_sn_cmd %p with sn %d added to skipped_sn_list" + " (expected_sn %d)", out_of_sn_cmd, out_of_sn_cmd->sn, + order_data->expected_sn); + spin_unlock_irq(&order_data->sn_lock); + } + + return; +} + +void scst_unblock_deferred(struct scst_order_data *order_data, + struct scst_cmd *out_of_sn_cmd) +{ + TRACE_ENTRY(); + + if (!out_of_sn_cmd->sn_set) { + TRACE_SN("cmd %p without sn", out_of_sn_cmd); + goto out; + } + + __scst_unblock_deferred(order_data, out_of_sn_cmd); + +out: + TRACE_EXIT(); + return; +} + /* dev_lock supposed to be held and BH disabled */ void scst_block_dev(struct scst_device *dev) { dev->block_count++; - TRACE_MGMT_DBG("Device BLOCK(new %d), dev %p", dev->block_count, dev); + TRACE_MGMT_DBG("Device BLOCK (new count %d), dev %s", dev->block_count, + dev->virt_name); } -/* No locks */ -void scst_unblock_dev(struct scst_device *dev) -{ - spin_lock_bh(&dev->dev_lock); - TRACE_MGMT_DBG("Device UNBLOCK(new %d), dev %p", - dev->block_count-1, dev); - if (--dev->block_count == 0) - scst_unblock_cmds(dev); - spin_unlock_bh(&dev->dev_lock); - sBUG_ON(dev->block_count < 0); -} - -/* No locks */ +/* dev_lock supposed to be held and BH disabled */ bool __scst_check_blocked_dev(struct scst_cmd *cmd) { int res = false; @@ -6602,144 +6646,123 @@ bool __scst_check_blocked_dev(struct scst_cmd *cmd) goto out; } -repeat: - if (dev->block_count > 0) { - spin_lock_bh(&dev->dev_lock); - if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) - goto out_unlock; - if (dev->block_count > 0) { - TRACE_MGMT_DBG("Delaying cmd %p due to blocking " - "(tag %llu, dev %p)", cmd, - (long long unsigned int)cmd->tag, dev); - list_add_tail(&cmd->blocked_cmd_list_entry, - &dev->blocked_cmd_list); - res = true; - spin_unlock_bh(&dev->dev_lock); - goto out; - } else { - TRACE_MGMT_DBG("%s", "Somebody unblocked the device, " - "continuing"); - } - spin_unlock_bh(&dev->dev_lock); - } + if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) + goto out; - if (dev->dev_double_ua_possible) { - spin_lock_bh(&dev->dev_lock); - if (dev->block_count == 0) { - TRACE_MGMT_DBG("cmd %p (tag %llu), blocking further " - "cmds due to possible double reset UA (dev %p)", - cmd, (long long unsigned int)cmd->tag, dev); - scst_block_dev(dev); + if (dev->block_count > 0) { + TRACE_MGMT_DBG("Delaying cmd %p due to blocking " + "(tag %llu, op %x, dev %s)", cmd, + (long long unsigned int)cmd->tag, cmd->cdb[0], + dev->virt_name); + goto out_block; + } else if (scst_is_strictly_serialized_cmd(cmd)) { + TRACE_MGMT_DBG("cmd %p (tag %llu, op %x): blocking further " + "cmds on dev %s due to strict serialization", cmd, + (long long unsigned int)cmd->tag, cmd->cdb[0], + dev->virt_name); + scst_block_dev(dev); + if (dev->on_dev_cmd_count > 1) { + TRACE_MGMT_DBG("Delaying strictly serialized cmd %p " + "(dev %s, on_dev_cmds to wait %d)", cmd, + dev->virt_name, dev->on_dev_cmd_count-1); + EXTRACHECKS_BUG_ON(dev->strictly_serialized_cmd_waiting); + dev->strictly_serialized_cmd_waiting = 1; + goto out_block; + } else cmd->unblock_dev = 1; - } else { - spin_unlock_bh(&dev->dev_lock); - TRACE_MGMT_DBG("Somebody blocked the device, " - "repeating (count %d)", dev->block_count); - goto repeat; - } - spin_unlock_bh(&dev->dev_lock); - } + } else if ((dev->dev_double_ua_possible) || scst_is_serialized_cmd(cmd)) { + TRACE_MGMT_DBG("cmd %p (tag %llu, op %x): blocking further cmds " + "on dev %s due to %s", cmd, (long long unsigned int)cmd->tag, + cmd->cdb[0], dev->virt_name, + dev->dev_double_ua_possible ? "possible double reset UA" : + "serialized cmd"); + scst_block_dev(dev); + cmd->unblock_dev = 1; + } else + TRACE_MGMT_DBG("No blocks for device %s", dev->virt_name); out: TRACE_EXIT_RES(res); return res; -out_unlock: - spin_unlock_bh(&dev->dev_lock); +out_block: + if (cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE) + list_add(&cmd->blocked_cmd_list_entry, + &dev->blocked_cmd_list); + else + list_add_tail(&cmd->blocked_cmd_list_entry, + &dev->blocked_cmd_list); + res = true; goto out; } -/* Called under dev_lock */ -static void scst_unblock_cmds(struct scst_device *dev) -{ - struct scst_cmd *cmd, *tcmd; - unsigned long flags; - - TRACE_ENTRY(); - - local_irq_save(flags); - list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list, - blocked_cmd_list_entry) { - list_del(&cmd->blocked_cmd_list_entry); - TRACE_MGMT_DBG("Adding blocked cmd %p to active cmd list", cmd); - spin_lock(&cmd->cmd_threads->cmd_list_lock); - if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE)) - list_add(&cmd->cmd_list_entry, - &cmd->cmd_threads->active_cmd_list); - else - list_add_tail(&cmd->cmd_list_entry, - &cmd->cmd_threads->active_cmd_list); - wake_up(&cmd->cmd_threads->cmd_list_waitQ); - spin_unlock(&cmd->cmd_threads->cmd_list_lock); - } - local_irq_restore(flags); - - TRACE_EXIT(); - return; -} - -static void __scst_unblock_deferred(struct scst_tgt_dev *tgt_dev, - struct scst_cmd *out_of_sn_cmd) -{ - EXTRACHECKS_BUG_ON(!out_of_sn_cmd->sn_set); - - if (out_of_sn_cmd->sn == tgt_dev->expected_sn) { - scst_inc_expected_sn(tgt_dev, out_of_sn_cmd->sn_slot); - scst_make_deferred_commands_active(tgt_dev); - } else { - out_of_sn_cmd->out_of_sn = 1; - spin_lock_irq(&tgt_dev->sn_lock); - tgt_dev->def_cmd_count++; - list_add_tail(&out_of_sn_cmd->sn_cmd_list_entry, - &tgt_dev->skipped_sn_list); - TRACE_SN("out_of_sn_cmd %p with sn %d added to skipped_sn_list" - " (expected_sn %d)", out_of_sn_cmd, out_of_sn_cmd->sn, - tgt_dev->expected_sn); - spin_unlock_irq(&tgt_dev->sn_lock); - } - - return; -} - -void scst_unblock_deferred(struct scst_tgt_dev *tgt_dev, - struct scst_cmd *out_of_sn_cmd) +/* dev_lock supposed to be held and BH disabled */ +void scst_unblock_dev(struct scst_device *dev) { TRACE_ENTRY(); - if (!out_of_sn_cmd->sn_set) { - TRACE_SN("cmd %p without sn", out_of_sn_cmd); - goto out; + TRACE_MGMT_DBG("Device UNBLOCK(new %d), dev %s", + dev->block_count-1, dev->virt_name); + + EXTRACHECKS_BUG_ON(!spin_is_locked(&dev->dev_lock)); + + if (--dev->block_count == 0) { + struct scst_cmd *cmd, *tcmd; + unsigned long flags; + + local_irq_save(flags); + list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list, + blocked_cmd_list_entry) { + bool strictly_serialized; + list_del(&cmd->blocked_cmd_list_entry); + TRACE_MGMT_DBG("Adding blocked cmd %p to active cmd " + "list", cmd); + spin_lock(&cmd->cmd_threads->cmd_list_lock); + if (cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE) + list_add(&cmd->cmd_list_entry, + &cmd->cmd_threads->active_cmd_list); + else + list_add_tail(&cmd->cmd_list_entry, + &cmd->cmd_threads->active_cmd_list); + strictly_serialized = scst_is_strictly_serialized_cmd(cmd); + wake_up(&cmd->cmd_threads->cmd_list_waitQ); + spin_unlock(&cmd->cmd_threads->cmd_list_lock); + if (dev->strictly_serialized_cmd_waiting && strictly_serialized) + break; + } + local_irq_restore(flags); + + dev->strictly_serialized_cmd_waiting = 0; } - __scst_unblock_deferred(tgt_dev, out_of_sn_cmd); + sBUG_ON(dev->block_count < 0); -out: TRACE_EXIT(); return; } void scst_on_hq_cmd_response(struct scst_cmd *cmd) { - struct scst_tgt_dev *tgt_dev = cmd->tgt_dev; + struct scst_order_data *order_data = cmd->cur_order_data; TRACE_ENTRY(); if (!cmd->hq_cmd_inced) goto out; - spin_lock_irq(&tgt_dev->sn_lock); - tgt_dev->hq_cmd_count--; - spin_unlock_irq(&tgt_dev->sn_lock); + spin_lock_irq(&order_data->sn_lock); + order_data->hq_cmd_count--; + spin_unlock_irq(&order_data->sn_lock); - EXTRACHECKS_BUG_ON(tgt_dev->hq_cmd_count < 0); + EXTRACHECKS_BUG_ON(order_data->hq_cmd_count < 0); /* * There is no problem in checking hq_cmd_count in the * non-locked state. In the worst case we will only have * unneeded run of the deferred commands. */ - if (tgt_dev->hq_cmd_count == 0) - scst_make_deferred_commands_active(tgt_dev); + if (order_data->hq_cmd_count == 0) + scst_make_deferred_commands_active(order_data); out: TRACE_EXIT(); diff --git a/scst/src/scst_main.c b/scst/src/scst_main.c index 278d049ce..d4cf5a99f 100644 --- a/scst/src/scst_main.c +++ b/scst/src/scst_main.c @@ -1571,8 +1571,8 @@ int scst_add_threads(struct scst_cmd_threads *cmd_threads, n++; } - TRACE_DBG("cmd_threads %p, dev %p, tgt_dev %p, num %d, n %d", - cmd_threads, dev, tgt_dev, num, n); + TRACE_DBG("cmd_threads %p, dev %s, tgt_dev %p, num %d, n %d", + cmd_threads, dev->virt_name, tgt_dev, num, n); if (tgt_dev != NULL) { struct scst_tgt_dev *t; @@ -2139,11 +2139,11 @@ static int __init init_scst(void) { struct scsi_sense_hdr *shdr; - struct scst_tgt_dev *t; + struct scst_order_data *o; struct scst_cmd *c; BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr)); - BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn)); - BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn)); + BUILD_BUG_ON(sizeof(o->curr_sn) != sizeof(o->expected_sn)); + BUILD_BUG_ON(sizeof(c->sn) != sizeof(o->expected_sn)); } mutex_init(&scst_mutex); diff --git a/scst/src/scst_pres.c b/scst/src/scst_pres.c index 3f51a8d64..b6a9db7f0 100644 --- a/scst/src/scst_pres.c +++ b/scst/src/scst_pres.c @@ -2458,7 +2458,7 @@ bool scst_pr_is_cmd_allowed(struct scst_cmd *cmd) TRACE_ENTRY(); - unlock = scst_pr_read_lock(dev); + unlock = scst_pr_read_lock(cmd); TRACE_DBG("Testing if command %s (0x%x) from %s allowed to execute", cmd->op_name, cmd->cdb[0], cmd->sess->initiator_name); @@ -2518,7 +2518,7 @@ bool scst_pr_is_cmd_allowed(struct scst_cmd *cmd) cmd->op_name, cmd->cdb[0], cmd->sess->initiator_name); out_unlock: - scst_pr_read_unlock(dev, unlock); + scst_pr_read_unlock(cmd, unlock); TRACE_EXIT_RES(allowed); return allowed; @@ -2540,7 +2540,7 @@ void scst_pr_read_keys(struct scst_cmd *cmd, uint8_t *buffer, int buffer_size) } TRACE_PR("Read Keys (dev %s): PRGen %d", dev->virt_name, - dev->pr_generation); + dev->pr_generation); put_unaligned(cpu_to_be32(dev->pr_generation), (__be32 *)&buffer[0]); diff --git a/scst/src/scst_pres.h b/scst/src/scst_pres.h index 1c7f07c8f..7fc6013a2 100644 --- a/scst/src/scst_pres.h +++ b/scst/src/scst_pres.h @@ -45,6 +45,52 @@ #define SCOPE_LU 0x00 +static inline void scst_inc_pr_readers_count(struct scst_cmd *cmd, + bool locked) +{ + struct scst_device *dev = cmd->dev; + + EXTRACHECKS_BUG_ON(cmd->dec_pr_readers_count_needed); + + if (!locked) + spin_lock_bh(&dev->dev_lock); + + EXTRACHECKS_BUG_ON(!spin_is_locked(&dev->dev_lock)); + + dev->pr_readers_count++; + cmd->dec_pr_readers_count_needed = 1; + TRACE_DBG("New inc pr_readers_count %d (cmd %p)", dev->pr_readers_count, + cmd); + + if (!locked) + spin_unlock_bh(&dev->dev_lock); + return; +} + +static inline void scst_dec_pr_readers_count(struct scst_cmd *cmd, + bool locked) +{ + struct scst_device *dev = cmd->dev; + + EXTRACHECKS_BUG_ON(!cmd->dec_pr_readers_count_needed); + + if (!locked) + spin_lock_bh(&dev->dev_lock); + + EXTRACHECKS_BUG_ON(!spin_is_locked(&dev->dev_lock)); + + dev->pr_readers_count--; + cmd->dec_pr_readers_count_needed = 0; + TRACE_DBG("New dec pr_readers_count %d (cmd %p)", dev->pr_readers_count, + cmd); + + if (!locked) + spin_unlock_bh(&dev->dev_lock); + + EXTRACHECKS_BUG_ON(dev->pr_readers_count < 0); + return; +} + static inline bool scst_pr_type_valid(uint8_t type) { switch (type) { @@ -60,18 +106,17 @@ static inline bool scst_pr_type_valid(uint8_t type) } } -static inline bool scst_pr_read_lock(struct scst_device *dev) +static inline bool scst_pr_read_lock(struct scst_cmd *cmd) { + struct scst_device *dev = cmd->dev; bool unlock = false; TRACE_ENTRY(); - atomic_inc(&dev->pr_readers_count); - smp_mb__after_atomic_inc(); /* to sync with scst_pr_write_lock() */ - + smp_mb(); /* to sync with scst_pr_write_lock() */ if (unlikely(dev->pr_writer_active)) { unlock = true; - atomic_dec(&dev->pr_readers_count); + scst_dec_pr_readers_count(cmd, false); mutex_lock(&dev->dev_pr_mutex); } @@ -79,20 +124,16 @@ static inline bool scst_pr_read_lock(struct scst_device *dev) return unlock; } -static inline void scst_pr_read_unlock(struct scst_device *dev, bool unlock) +static inline void scst_pr_read_unlock(struct scst_cmd *cmd, bool unlock) { + struct scst_device *dev = cmd->dev; + TRACE_ENTRY(); if (unlikely(unlock)) mutex_unlock(&dev->dev_pr_mutex); - else { - /* - * To sync with scst_pr_write_lock(). We need it to ensure - * order of our reads with the writer's writes. - */ - smp_mb__before_atomic_dec(); - atomic_dec(&dev->pr_readers_count); - } + else + scst_dec_pr_readers_count(cmd, false); TRACE_EXIT(); return; @@ -105,13 +146,17 @@ static inline void scst_pr_write_lock(struct scst_device *dev) mutex_lock(&dev->dev_pr_mutex); dev->pr_writer_active = 1; - /* to sync with scst_pr_read_lock() and unlock() */ smp_mb(); - while (atomic_read(&dev->pr_readers_count) != 0) { - TRACE_DBG("Waiting for %d readers (dev %p)", - atomic_read(&dev->pr_readers_count), dev); + while (true) { + int readers; + spin_lock_bh(&dev->dev_lock); + readers = dev->pr_readers_count; + spin_unlock_bh(&dev->dev_lock); + if (readers == 0) + break; + TRACE_DBG("Waiting for %d readers (dev %p)", readers, dev); msleep(1); } @@ -124,7 +169,6 @@ static inline void scst_pr_write_unlock(struct scst_device *dev) TRACE_ENTRY(); dev->pr_writer_active = 0; - mutex_unlock(&dev->dev_pr_mutex); TRACE_EXIT(); diff --git a/scst/src/scst_priv.h b/scst/src/scst_priv.h index db6603b02..ab31e5386 100644 --- a/scst/src/scst_priv.h +++ b/scst/src/scst_priv.h @@ -251,24 +251,24 @@ extern bool scst_del_thr_data(struct scst_tgt_dev *tgt_dev, extern struct scst_dev_type scst_null_devtype; extern struct scst_cmd *__scst_check_deferred_commands( - struct scst_tgt_dev *tgt_dev); + struct scst_order_data *order_data); /* Used to save the function call on the fast path */ static inline struct scst_cmd *scst_check_deferred_commands( - struct scst_tgt_dev *tgt_dev) + struct scst_order_data *order_data) { - if (tgt_dev->def_cmd_count == 0) + if (order_data->def_cmd_count == 0) return NULL; else - return __scst_check_deferred_commands(tgt_dev); + return __scst_check_deferred_commands(order_data); } static inline void scst_make_deferred_commands_active( - struct scst_tgt_dev *tgt_dev) + struct scst_order_data *order_data) { struct scst_cmd *c; - c = __scst_check_deferred_commands(tgt_dev); + c = __scst_check_deferred_commands(order_data); if (c != NULL) { TRACE_SN("Adding cmd %p to active cmd list", c); spin_lock_irq(&c->cmd_threads->cmd_list_lock); @@ -281,10 +281,10 @@ static inline void scst_make_deferred_commands_active( return; } -void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot); +void scst_inc_expected_sn(struct scst_order_data *order_data, atomic_t *slot); int scst_check_hq_cmd(struct scst_cmd *cmd); -void scst_unblock_deferred(struct scst_tgt_dev *tgt_dev, +void scst_unblock_deferred(struct scst_order_data *order_data, struct scst_cmd *cmd_sn); void scst_on_hq_cmd_response(struct scst_cmd *cmd); @@ -537,43 +537,32 @@ void scst_free_aen(struct scst_aen *aen); void scst_gen_aen_or_ua(struct scst_tgt_dev *tgt_dev, int key, int asc, int ascq); -static inline bool scst_is_implicit_hq(struct scst_cmd *cmd) +static inline bool scst_is_implicit_hq_cmd(struct scst_cmd *cmd) { return (cmd->op_flags & SCST_IMPLICIT_HQ) != 0; } +static inline bool scst_is_serialized_cmd(struct scst_cmd *cmd) +{ + return (cmd->op_flags & SCST_SERIALIZED) != 0; +} + +static inline bool scst_is_strictly_serialized_cmd(struct scst_cmd *cmd) +{ + return (cmd->op_flags & SCST_STRICTLY_SERIALIZED) == SCST_STRICTLY_SERIALIZED; +} + /* * Some notes on devices "blocking". Blocking means that no * commands will go from SCST to underlying SCSI device until it - * is unblocked. But we don't care about all commands that - * already on the device. + * is unblocked. But, except for strictly serialized commands, + * we don't care about all commands that already on the device. */ extern void scst_block_dev(struct scst_device *dev); extern void scst_unblock_dev(struct scst_device *dev); -extern bool __scst_check_blocked_dev(struct scst_cmd *cmd); - -static inline bool scst_check_blocked_dev(struct scst_cmd *cmd) -{ - if (unlikely(cmd->dev->block_count > 0) || - unlikely(cmd->dev->dev_double_ua_possible)) - return __scst_check_blocked_dev(cmd); - else - return false; -} - -/* No locks */ -static inline void scst_check_unblock_dev(struct scst_cmd *cmd) -{ - if (unlikely(cmd->unblock_dev)) { - TRACE_MGMT_DBG("cmd %p (tag %llu): unblocking dev %p", cmd, - (long long unsigned int)cmd->tag, cmd->dev); - cmd->unblock_dev = 0; - scst_unblock_dev(cmd->dev); - } - return; -} +bool __scst_check_blocked_dev(struct scst_cmd *cmd); /* * Increases global SCST ref counters which prevent from entering into suspended diff --git a/scst/src/scst_targ.c b/scst/src/scst_targ.c index 4b51c1a5f..6c28a1258 100644 --- a/scst/src/scst_targ.c +++ b/scst/src/scst_targ.c @@ -102,6 +102,77 @@ static inline void scst_schedule_tasklet(struct scst_cmd *cmd) return; } +/* No locks */ +static bool scst_check_blocked_dev(struct scst_cmd *cmd) +{ + bool res; + struct scst_device *dev = cmd->dev; + + spin_lock_bh(&dev->dev_lock); + + dev->on_dev_cmd_count++; + cmd->dec_on_dev_needed = 1; + TRACE_DBG("New inc on_dev_count %d (cmd %p)", dev->on_dev_cmd_count, + cmd); + + scst_inc_pr_readers_count(cmd, true); + + if (unlikely(dev->block_count > 0) || + unlikely(dev->dev_double_ua_possible) || + unlikely(scst_is_serialized_cmd(cmd))) + res = __scst_check_blocked_dev(cmd); + else + res = false; + + if (unlikely(res)) { + /* Undo increments */ + dev->on_dev_cmd_count--; + cmd->dec_on_dev_needed = 0; + TRACE_DBG("New dec on_dev_count %d (cmd %p)", + dev->on_dev_cmd_count, cmd); + + scst_dec_pr_readers_count(cmd, true); + } + + spin_unlock_bh(&dev->dev_lock); + + return res; +} + +/* No locks */ +static void scst_check_unblock_dev(struct scst_cmd *cmd) +{ + struct scst_device *dev = cmd->dev; + + spin_lock_bh(&dev->dev_lock); + + if (likely(cmd->dec_on_dev_needed)) { + dev->on_dev_cmd_count--; + cmd->dec_on_dev_needed = 0; + TRACE_DBG("New dec on_dev_count %d (cmd %p)", + dev->on_dev_cmd_count, cmd); + } + + if (unlikely(cmd->dec_pr_readers_count_needed)) + scst_dec_pr_readers_count(cmd, true); + + if (unlikely(cmd->unblock_dev)) { + TRACE_MGMT_DBG("cmd %p (tag %llu): unblocking dev %s", cmd, + (long long unsigned int)cmd->tag, dev->virt_name); + cmd->unblock_dev = 0; + scst_unblock_dev(dev); + } else if (unlikely(dev->strictly_serialized_cmd_waiting)) { + if (dev->on_dev_cmd_count == 0) { + TRACE_MGMT_DBG("Strictly serialized cmd waiting: " + "unblocking dev %s", dev->virt_name); + scst_unblock_dev(dev); + } + } + + spin_unlock_bh(&dev->dev_lock); + return; +} + /** * scst_rx_cmd() - create new command * @sess: SCST session @@ -2304,7 +2375,7 @@ out_done: * !! Dev handlers implementing exec() callback must call this function there * !! just before the actual command's execution! * - * On call no locks, no IRQ or IRQ-disabled context allowed. + * On call no locks, no IRQ or IRQ-disabled context allowed. */ int scst_check_local_events(struct scst_cmd *cmd) { @@ -2327,7 +2398,7 @@ int scst_check_local_events(struct scst_cmd *cmd) if ((cmd->op_flags & SCST_REG_RESERVE_ALLOWED) == 0) { scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT); - goto out_complete; + goto out_dec_pr_readers_count; } } @@ -2337,7 +2408,8 @@ int scst_check_local_events(struct scst_cmd *cmd) SAM_STAT_RESERVATION_CONFLICT); goto out_complete; } - } + } else + scst_dec_pr_readers_count(cmd, false); /* * Let's check for ABORTED after scst_pr_is_cmd_allowed(), because @@ -2354,8 +2426,7 @@ int scst_check_local_events(struct scst_cmd *cmd) if (scst_is_ua_command(cmd)) { int done = 0; /* - * Prevent more than 1 cmd to be triggered by - * was_reset. + * Prevent more than 1 cmd to be triggered by was_reset */ spin_lock_bh(&dev->dev_lock); if (dev->scsi_dev->was_reset) { @@ -2364,7 +2435,7 @@ int scst_check_local_events(struct scst_cmd *cmd) SCST_LOAD_SENSE(scst_sense_reset_UA)); /* * It looks like it is safe to clear was_reset - * here. + * here */ dev->scsi_dev->was_reset = 0; done = 1; @@ -2391,6 +2462,10 @@ out: TRACE_EXIT_RES(res); return res; +out_dec_pr_readers_count: + if (cmd->dec_pr_readers_count_needed) + scst_dec_pr_readers_count(cmd, false); + out_complete: res = 1; sBUG_ON(!cmd->completed); @@ -2403,36 +2478,34 @@ out_uncomplete: EXPORT_SYMBOL_GPL(scst_check_local_events); /* No locks */ -void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot) +void scst_inc_expected_sn(struct scst_order_data *order_data, atomic_t *slot) { if (slot == NULL) goto inc; /* Optimized for lockless fast path */ - TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots, + TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - order_data->sn_slots, atomic_read(slot)); if (!atomic_dec_and_test(slot)) goto out; TRACE_SN("Slot is 0 (num_free_sn_slots=%d)", - tgt_dev->num_free_sn_slots); - if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) { - spin_lock_irq(&tgt_dev->sn_lock); - if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) { - if (tgt_dev->num_free_sn_slots < 0) - tgt_dev->cur_sn_slot = slot; - /* - * To be in-sync with SIMPLE case in scst_cmd_set_sn() - */ + order_data->num_free_sn_slots); + if (order_data->num_free_sn_slots < (int)ARRAY_SIZE(order_data->sn_slots)-1) { + spin_lock_irq(&order_data->sn_lock); + if (likely(order_data->num_free_sn_slots < (int)ARRAY_SIZE(order_data->sn_slots)-1)) { + if (order_data->num_free_sn_slots < 0) + order_data->cur_sn_slot = slot; + /* To be in-sync with SIMPLE case in scst_cmd_set_sn() */ smp_mb(); - tgt_dev->num_free_sn_slots++; + order_data->num_free_sn_slots++; TRACE_SN("Incremented num_free_sn_slots (%d)", - tgt_dev->num_free_sn_slots); + order_data->num_free_sn_slots); } - spin_unlock_irq(&tgt_dev->sn_lock); + spin_unlock_irq(&order_data->sn_lock); } inc: @@ -2441,13 +2514,13 @@ inc: * at time can be here (serialized by sn). Also it is supposed that * there could not be half-incremented halves. */ - tgt_dev->expected_sn++; + order_data->expected_sn++; /* * Write must be before def_cmd_count read to be in sync. with * scst_post_exec_sn(). See comment in scst_send_for_exec(). */ smp_mb(); - TRACE_SN("Next expected_sn: %d", tgt_dev->expected_sn); + TRACE_SN("Next expected_sn: %d", order_data->expected_sn); out: return; @@ -2460,19 +2533,19 @@ static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd, /* For HQ commands SN is not set */ bool inc_expected_sn = !cmd->inc_expected_sn_on_done && cmd->sn_set && !cmd->retry; - struct scst_tgt_dev *tgt_dev = cmd->tgt_dev; + struct scst_order_data *order_data = cmd->cur_order_data; struct scst_cmd *res; TRACE_ENTRY(); if (inc_expected_sn) - scst_inc_expected_sn(tgt_dev, cmd->sn_slot); + scst_inc_expected_sn(order_data, cmd->sn_slot); if (make_active) { - scst_make_deferred_commands_active(tgt_dev); + scst_make_deferred_commands_active(order_data); res = NULL; } else - res = scst_check_deferred_commands(tgt_dev); + res = scst_check_deferred_commands(order_data); TRACE_EXIT_HRES(res); return res; @@ -2766,8 +2839,8 @@ static int scst_send_for_exec(struct scst_cmd **active_cmd) { int res; struct scst_cmd *cmd = *active_cmd; - struct scst_tgt_dev *tgt_dev = cmd->tgt_dev; - typeof(tgt_dev->expected_sn) expected_sn; + struct scst_order_data *order_data = cmd->cur_order_data; + typeof(order_data->expected_sn) expected_sn; TRACE_ENTRY(); @@ -2779,12 +2852,12 @@ static int scst_send_for_exec(struct scst_cmd **active_cmd) sBUG_ON(!cmd->sn_set); - expected_sn = tgt_dev->expected_sn; + expected_sn = order_data->expected_sn; /* Optimized for lockless fast path */ - if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) { - spin_lock_irq(&tgt_dev->sn_lock); + if ((cmd->sn != expected_sn) || (order_data->hq_cmd_count > 0)) { + spin_lock_irq(&order_data->sn_lock); - tgt_dev->def_cmd_count++; + order_data->def_cmd_count++; /* * Memory barrier is needed here to implement lockless fast * path. We need the exact order of read and write between @@ -2798,15 +2871,15 @@ static int scst_send_for_exec(struct scst_cmd **active_cmd) */ smp_mb(); - expected_sn = tgt_dev->expected_sn; - if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) { + expected_sn = order_data->expected_sn; + if ((cmd->sn != expected_sn) || (order_data->hq_cmd_count > 0)) { if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) { /* Necessary to allow aborting out of sn cmds */ TRACE_MGMT_DBG("Aborting out of sn cmd %p " "(tag %llu, sn %u)", cmd, (long long unsigned)cmd->tag, cmd->sn); - tgt_dev->def_cmd_count--; + order_data->def_cmd_count--; scst_set_cmd_abnormal_done_state(cmd); res = SCST_CMD_STATE_RES_CONT_SAME; } else { @@ -2814,16 +2887,16 @@ static int scst_send_for_exec(struct scst_cmd **active_cmd) "expected_sn=%d)", cmd, cmd->sn, cmd->sn_set, expected_sn); list_add_tail(&cmd->sn_cmd_list_entry, - &tgt_dev->deferred_cmd_list); + &order_data->deferred_cmd_list); res = SCST_CMD_STATE_RES_CONT_NEXT; } - spin_unlock_irq(&tgt_dev->sn_lock); + spin_unlock_irq(&order_data->sn_lock); goto out; } else { TRACE_SN("Somebody incremented expected_sn %d, " "continuing", expected_sn); - tgt_dev->def_cmd_count--; - spin_unlock_irq(&tgt_dev->sn_lock); + order_data->def_cmd_count--; + spin_unlock_irq(&order_data->sn_lock); } } @@ -3204,9 +3277,9 @@ out: static void scst_inc_check_expected_sn(struct scst_cmd *cmd) { if (likely(cmd->sn_set)) - scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot); + scst_inc_expected_sn(cmd->cur_order_data, cmd->sn_slot); - scst_make_deferred_commands_active(cmd->tgt_dev); + scst_make_deferred_commands_active(cmd->cur_order_data); } static int scst_dev_done(struct scst_cmd *cmd) @@ -3361,7 +3434,7 @@ static int scst_pre_xmit_response(struct scst_cmd *cmd) TRACE_SN("cmd %p was not sent to mid-lev" " (sn %d, set %d)", cmd, cmd->sn, cmd->sn_set); - scst_unblock_deferred(cmd->tgt_dev, cmd); + scst_unblock_deferred(cmd->cur_order_data, cmd); cmd->sent_for_exec = 1; } } @@ -3621,12 +3694,12 @@ out: */ static void scst_cmd_set_sn(struct scst_cmd *cmd) { - struct scst_tgt_dev *tgt_dev = cmd->tgt_dev; + struct scst_order_data *order_data = cmd->cur_order_data; unsigned long flags; TRACE_ENTRY(); - if (scst_is_implicit_hq(cmd) && + if (scst_is_implicit_hq_cmd(cmd) && likely(cmd->queue_type == SCST_CMD_QUEUE_SIMPLE)) { TRACE_SN("Implicit HQ cmd %p", cmd); cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE; @@ -3654,23 +3727,23 @@ static void scst_cmd_set_sn(struct scst_cmd *cmd) switch (cmd->queue_type) { case SCST_CMD_QUEUE_SIMPLE: case SCST_CMD_QUEUE_UNTAGGED: - if (likely(tgt_dev->num_free_sn_slots >= 0)) { + if (likely(order_data->num_free_sn_slots >= 0)) { /* * atomic_inc_return() implies memory barrier to sync * with scst_inc_expected_sn() */ - if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) { - tgt_dev->curr_sn++; + if (atomic_inc_return(order_data->cur_sn_slot) == 1) { + order_data->curr_sn++; TRACE_SN("Incremented curr_sn %d", - tgt_dev->curr_sn); + order_data->curr_sn); } - cmd->sn_slot = tgt_dev->cur_sn_slot; - cmd->sn = tgt_dev->curr_sn; + cmd->sn_slot = order_data->cur_sn_slot; + cmd->sn = order_data->curr_sn; - tgt_dev->prev_cmd_ordered = 0; + order_data->prev_cmd_ordered = 0; } else { TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots " - "%zd", ARRAY_SIZE(tgt_dev->sn_slots)); + "%zd", ARRAY_SIZE(order_data->sn_slots)); goto ordered; } break; @@ -3678,44 +3751,44 @@ static void scst_cmd_set_sn(struct scst_cmd *cmd) case SCST_CMD_QUEUE_ORDERED: TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]); ordered: - if (!tgt_dev->prev_cmd_ordered) { - spin_lock_irqsave(&tgt_dev->sn_lock, flags); - if (tgt_dev->num_free_sn_slots >= 0) { - tgt_dev->num_free_sn_slots--; - if (tgt_dev->num_free_sn_slots >= 0) { + if (!order_data->prev_cmd_ordered) { + spin_lock_irqsave(&order_data->sn_lock, flags); + if (order_data->num_free_sn_slots >= 0) { + order_data->num_free_sn_slots--; + if (order_data->num_free_sn_slots >= 0) { int i = 0; /* Commands can finish in any order, so * we don't know which slot is empty. */ while (1) { - tgt_dev->cur_sn_slot++; - if (tgt_dev->cur_sn_slot == - tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots)) - tgt_dev->cur_sn_slot = tgt_dev->sn_slots; + order_data->cur_sn_slot++; + if (order_data->cur_sn_slot == + order_data->sn_slots + ARRAY_SIZE(order_data->sn_slots)) + order_data->cur_sn_slot = order_data->sn_slots; - if (atomic_read(tgt_dev->cur_sn_slot) == 0) + if (atomic_read(order_data->cur_sn_slot) == 0) break; i++; - sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots)); + sBUG_ON(i == ARRAY_SIZE(order_data->sn_slots)); } TRACE_SN("New cur SN slot %zd", - tgt_dev->cur_sn_slot - - tgt_dev->sn_slots); + order_data->cur_sn_slot - + order_data->sn_slots); } } - spin_unlock_irqrestore(&tgt_dev->sn_lock, flags); + spin_unlock_irqrestore(&order_data->sn_lock, flags); } - tgt_dev->prev_cmd_ordered = 1; - tgt_dev->curr_sn++; - cmd->sn = tgt_dev->curr_sn; + order_data->prev_cmd_ordered = 1; + order_data->curr_sn++; + cmd->sn = order_data->curr_sn; break; case SCST_CMD_QUEUE_HEAD_OF_QUEUE: TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]); - spin_lock_irqsave(&tgt_dev->sn_lock, flags); - tgt_dev->hq_cmd_count++; - spin_unlock_irqrestore(&tgt_dev->sn_lock, flags); + spin_lock_irqsave(&order_data->sn_lock, flags); + order_data->hq_cmd_count++; + spin_unlock_irqrestore(&order_data->sn_lock, flags); cmd->hq_cmd_inced = 1; goto out; @@ -3723,12 +3796,12 @@ ordered: sBUG(); } - TRACE_SN("cmd(%p)->sn: %d (tgt_dev %p, *cur_sn_slot %d, " + TRACE_SN("cmd(%p)->sn: %d (order_data %p, *cur_sn_slot %d, " "num_free_sn_slots %d, prev_cmd_ordered %ld, " - "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev, - atomic_read(tgt_dev->cur_sn_slot), - tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered, - tgt_dev->cur_sn_slot-tgt_dev->sn_slots); + "cur_sn_slot %zd)", cmd, cmd->sn, order_data, + atomic_read(order_data->cur_sn_slot), + order_data->num_free_sn_slots, order_data->prev_cmd_ordered, + order_data->cur_sn_slot - order_data->sn_slots); cmd->sn_set = 1; @@ -3774,6 +3847,7 @@ static int scst_translate_lun(struct scst_cmd *cmd) cmd->cmd_threads = tgt_dev->active_cmd_threads; cmd->tgt_dev = tgt_dev; + cmd->cur_order_data = tgt_dev->curr_order_data; cmd->dev = tgt_dev->dev; res = 0; @@ -4830,20 +4904,21 @@ static void scst_unblock_aborted_cmds(int scst_mutex_held) local_irq_disable(); list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, - dev_tgt_dev_list_entry) { - spin_lock(&tgt_dev->sn_lock); + dev_tgt_dev_list_entry) { + struct scst_order_data *order_data = tgt_dev->curr_order_data; + spin_lock(&order_data->sn_lock); list_for_each_entry_safe(cmd, tcmd, - &tgt_dev->deferred_cmd_list, + &order_data->deferred_cmd_list, sn_cmd_list_entry) { if (__scst_check_unblock_aborted_cmd(cmd, &cmd->sn_cmd_list_entry)) { TRACE_MGMT_DBG("Unblocked aborted SN " "cmd %p (sn %u)", cmd, cmd->sn); - tgt_dev->def_cmd_count--; + order_data->def_cmd_count--; } } - spin_unlock(&tgt_dev->sn_lock); + spin_unlock(&order_data->sn_lock); } local_irq_enable(); } @@ -4935,13 +5010,13 @@ static int scst_is_cmd_belongs_to_dev(struct scst_cmd *cmd, TRACE_ENTRY(); - TRACE_DBG("Finding match for dev %p and cmd %p (lun %lld)", dev, cmd, - (long long unsigned int)cmd->lun); + TRACE_DBG("Finding match for dev %s and cmd %p (lun %lld)", dev->virt_name, + cmd, (long long unsigned int)cmd->lun); head = &cmd->sess->sess_tgt_dev_list[SESS_TGT_DEV_LIST_HASH_FN(cmd->lun)]; list_for_each_entry(tgt_dev, head, sess_tgt_dev_list_entry) { if (tgt_dev->lun == cmd->lun) { - TRACE_DBG("dev %p found", tgt_dev->dev); + TRACE_DBG("dev %s found", tgt_dev->dev->virt_name); res = (tgt_dev->dev == dev); goto out; } @@ -5615,7 +5690,10 @@ static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd) switch (mcmd->fn) { case SCST_LUN_RESET: case SCST_CLEAR_TASK_SET: - scst_unblock_dev(mcmd->mcmd_tgt_dev->dev); + dev = mcmd->mcmd_tgt_dev->dev; + spin_lock_bh(&dev->dev_lock); + scst_unblock_dev(dev); + spin_unlock_bh(&dev->dev_lock); break; case SCST_TARGET_RESET: @@ -5627,7 +5705,9 @@ static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd) list_for_each_entry(acg_dev, &acg->acg_dev_list, acg_dev_list_entry) { dev = acg_dev->dev; + spin_lock_bh(&dev->dev_lock); scst_unblock_dev(dev); + spin_unlock_bh(&dev->dev_lock); } mutex_unlock(&scst_mutex); break;