mirrors/scst
1
0
Fork 0
mirror of https://github.com/SCST-project/scst.git synced 2026-05-28 17:30:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Commands ordering/serialization improvements

Browse Source 
git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@3382 d57e44dd-8a1f-0410-8b47-8ef2f437770f
This commit is contained in:
Vladislav Bolkhovitin
2011-04-13 17:18:27 +00:00
parent 1f049c1f8c
commit d0928114c0
8 changed files with 510 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
scst/include/scst.h
View File

@@ -1691,6 +1691,34 @@ struct scst_cmd_threads {
struct list_head lists_list_entry;
};
/*
* Used to execute cmd's in order of arrival, honoring SCSI task attributes
*/
struct scst_order_data {
/*
* Protected by sn_lock, except expected_sn, which is protected by
* itself. Curr_sn must have the same size as expected_sn to
* overflow simultaneously.
*/
int def_cmd_count;
spinlock_t sn_lock;
unsigned int expected_sn;
unsigned int curr_sn;
int hq_cmd_count;
struct list_head deferred_cmd_list;
struct list_head skipped_sn_list;
/*
* Set if the prev cmd was ORDERED. Size and, hence, alignment must
* allow unprotected modifications independently to the neighbour fields.
*/
unsigned long prev_cmd_ordered;
int num_free_sn_slots; /* if it's <0, then all slots are busy */
atomic_t *cur_sn_slot;
atomic_t sn_slots[15];
};
/*
* SCST command, analog of I_T_L_Q nexus or task
*/
@@ -1744,6 +1772,12 @@ struct scst_cmd {
/* Set if the device was blocked by scst_check_blocked_dev() */
unsigned int unblock_dev:1;
/* Set if this cmd incremented dev->pr_readers_count */
unsigned int dec_pr_readers_count_needed:1;
/* Set if scst_dec_on_dev_cmd() call is needed on the cmd's finish */
unsigned int dec_on_dev_needed:1;
/* Set if cmd is queued as hw pending */
unsigned int cmd_hw_pending:1;
@@ -1851,7 +1885,10 @@ struct scst_cmd {
struct scst_tgt *tgt; /* to save extra dereferences */
struct scst_device *dev; /* to save extra dereferences */
struct scst_tgt_dev *tgt_dev; /* corresponding device for this cmd */
/* corresponding I_T_L device for this cmd */
struct scst_tgt_dev *tgt_dev;
struct scst_order_data *cur_order_data; /* to save extra dereferences */
uint64_t lun; /* LUN for this cmd */
@@ -2148,6 +2185,9 @@ struct scst_device {
/* If set, dev is read only */
unsigned short rd_only:1;
/* Set, if a strictly serialized cmd is waiting blocked */
unsigned short strictly_serialized_cmd_waiting:1;
/**************************************************************/
/*************************************************************
@@ -2171,14 +2211,28 @@ struct scst_device {
/**************************************************************/
/* How many cmds alive on this dev */
atomic_t dev_cmd_count;
spinlock_t dev_lock; /* device lock */
/*
* How many times device was blocked for new cmds execution.
* Protected by dev_lock
* Protected by dev_lock.
*/
int block_count;
/* How many cmds alive on this dev */
atomic_t dev_cmd_count;
/*
* How many there are "on_dev" commands, i.e. ones who passed
* scst_check_blocked_dev(). Protected by dev_lock.
*/
int on_dev_cmd_count;
/*
* How many threads are checking commands for PR allowance.
* Protected by dev_lock.
*/
int pr_readers_count;
/*
* Set if dev is persistently reserved. Protected by dev_pr_mutex.
@@ -2192,12 +2246,6 @@ struct scst_device {
*/
unsigned int pr_writer_active:1;
/*
* How many threads are checking commands for PR allowance. Used to
* implement lockless read-only fast path.
*/
atomic_t pr_readers_count;
struct scst_dev_type *handler; /* corresponding dev handler */
/* Used for storage of dev handler private stuff */
@@ -2247,27 +2295,26 @@ struct scst_device {
*/
int not_pr_supporting_tgt_devs_num;
struct scst_order_data dev_order_data;
/* Persist through power loss files */
char *pr_file_name;
char *pr_file_name1;
/**************************************************************/
spinlock_t dev_lock; /* device lock */
struct list_head blocked_cmd_list; /* protected by dev_lock */
/* List of blocked commands, protected by dev_lock. */
struct list_head blocked_cmd_list;
/* A list entry used during TM, protected by scst_mutex */
struct list_head tm_dev_list_entry;
/* Virtual device internal ID */
int virt_id;
int virt_id; /* virtual device internal ID */
/* Pointer to virtual device name, for convenience only */
char *virt_name;
/* List entry in global devices list */
struct list_head dev_list_entry;
struct list_head dev_list_entry; /* list entry in global devices list */
/*
* List of tgt_dev's, one per session, protected by scst_mutex or
@@ -2346,31 +2393,8 @@ struct scst_tgt_dev {
/* How many cmds alive on this dev in this session */
atomic_t tgt_dev_cmd_count;
/*
* Used to execute cmd's in order of arrival, honoring SCSI task
* attributes.
*
* Protected by sn_lock, except expected_sn, which is protected by
* itself. Curr_sn must have the same size as expected_sn to
* overflow simultaneously.
*/
int def_cmd_count;
spinlock_t sn_lock;
unsigned int expected_sn;
unsigned int curr_sn;
int hq_cmd_count;
struct list_head deferred_cmd_list;
struct list_head skipped_sn_list;
/*
* Set if the prev cmd was ORDERED. Size and, hence, alignment must
* allow unprotected modifications independently to the neighbour fields.
*/
unsigned long prev_cmd_ordered;
int num_free_sn_slots; /* if it's <0, then all slots are busy */
atomic_t *cur_sn_slot;
atomic_t sn_slots[15];
struct scst_order_data *curr_order_data;
struct scst_order_data tgt_dev_order_data;
/* List of scst_thr_data_hdr and lock */
spinlock_t thr_data_lock;

8
scst/include/scst_const.h
View File

@@ -175,10 +175,6 @@ enum scst_cmd_queue_type {
/*************************************************************
** CDB flags
**
** Implicit ordered used for commands which need calm environment
** without any simultaneous activities. For instance, for MODE
** SELECT it is needed to correctly generate its UA.
*************************************************************/
enum scst_cdb_flags {
SCST_TRANSFER_LEN_TYPE_FIXED = 0x0001,
@@ -196,8 +192,10 @@ enum scst_cdb_flags {
SCST_WRITE_EXCL_ALLOWED = 0x1000,
SCST_EXCL_ACCESS_ALLOWED = 0x2000,
#ifdef CONFIG_SCST_TEST_IO_IN_SIRQ
SCST_TEST_IO_IN_SIRQ_ALLOWED = 0x8000,
SCST_TEST_IO_IN_SIRQ_ALLOWED = 0x4000,
#endif
SCST_SERIALIZED = 0x8000,
SCST_STRICTLY_SERIALIZED = 0x10000|SCST_SERIALIZED,
};
/*************************************************************

341
scst/src/scst_lib.c
View File

@@ -134,7 +134,7 @@ struct scst_sdbops {
uint8_t direction; /* init --> target: SCST_DATA_WRITE
* target --> init: SCST_DATA_READ
*/
uint16_t flags; /* opcode -- various flags */
uint32_t flags; /* opcode -- various flags */
uint8_t off; /* length offset in cdb */
int (*get_trans_len)(struct scst_cmd *cmd, uint8_t off);
};
@@ -271,13 +271,13 @@ static const struct scst_sdbops scst_scsi_op_table[] = {
SCST_WRITE_EXCL_ALLOWED,
2, get_trans_len_3},
{0x15, "OMOOOOOOOOOOOOOO", "MODE SELECT(6)",
SCST_DATA_WRITE, FLAG_NONE, 4, get_trans_len_1},
SCST_DATA_WRITE, SCST_STRICTLY_SERIALIZED, 4, get_trans_len_1},
{0x16, "MMMMMMMMMMMMMMMM", "RESERVE",
SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|
SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED|
SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED,
0, get_trans_len_none},
{0x17, "MMMMMMMMMMMMMMMM", "RELEASE",
SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|
SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED|
SCST_REG_RESERVE_ALLOWED|
SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED,
0, get_trans_len_none},
@@ -433,7 +433,7 @@ static const struct scst_sdbops scst_scsi_op_table[] = {
{0x4B, " O ", "PAUSE/RESUME",
SCST_DATA_NONE, FLAG_NONE, 0, get_trans_len_none},
{0x4C, "OOOOOOOOOOOOOOOO", "LOG SELECT",
SCST_DATA_WRITE, FLAG_NONE, 7, get_trans_len_2},
SCST_DATA_WRITE, SCST_STRICTLY_SERIALIZED, 7, get_trans_len_2},
{0x4D, "OOOOOOOOOOOOOOOO", "LOG SENSE",
SCST_DATA_READ, SCST_SMALL_TIMEOUT|
SCST_REG_RESERVE_ALLOWED|
@@ -459,13 +459,15 @@ static const struct scst_sdbops scst_scsi_op_table[] = {
{0x54, " O ", "SEND OPC INFORMATION",
SCST_DATA_WRITE, FLAG_NONE, 7, get_trans_len_2},
{0x55, "OOOOOOOOOOOOOOOO", "MODE SELECT(10)",
SCST_DATA_WRITE, FLAG_NONE, 7, get_trans_len_2},
SCST_DATA_WRITE, SCST_STRICTLY_SERIALIZED, 7, get_trans_len_2},
{0x56, "OOOOOOOOOOOOOOOO", "RESERVE(10)",
SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD,
SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED|
SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED,
0, get_trans_len_none},
{0x57, "OOOOOOOOOOOOOOOO", "RELEASE(10)",
SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|
SCST_REG_RESERVE_ALLOWED,
SCST_DATA_NONE, SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED|
SCST_REG_RESERVE_ALLOWED|
SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED,
0, get_trans_len_none},
{0x58, " O ", "REPAIR TRACK",
SCST_DATA_NONE, SCST_WRITE_MEDIUM, 0, get_trans_len_none},
@@ -477,15 +479,15 @@ static const struct scst_sdbops scst_scsi_op_table[] = {
SCST_DATA_READ, FLAG_NONE, 7, get_trans_len_2},
{0x5D, " O ", "SEND CUE SHEET",
SCST_DATA_WRITE, FLAG_NONE, 6, get_trans_len_3},
{0x5E, "OOOOO OOOO ", "PERSISTENT RESERV IN",
{0x5E, "OOOOO OOOO ", "PERSISTENT RESERVE IN",
SCST_DATA_READ, SCST_SMALL_TIMEOUT|
SCST_LOCAL_CMD|
SCST_LOCAL_CMD|SCST_SERIALIZED|
SCST_WRITE_EXCL_ALLOWED|
SCST_EXCL_ACCESS_ALLOWED,
5, get_trans_len_4},
{0x5F, "OOOOO OOOO ", "PERSISTENT RESERV OUT",
{0x5F, "OOOOO OOOO ", "PERSISTENT RESERVE OUT",
SCST_DATA_WRITE, SCST_SMALL_TIMEOUT|
SCST_LOCAL_CMD|
SCST_LOCAL_CMD|SCST_SERIALIZED|
SCST_WRITE_EXCL_ALLOWED|
SCST_EXCL_ACCESS_ALLOWED,
5, get_trans_len_4},
@@ -688,7 +690,6 @@ static void scst_alloc_set_UA(struct scst_tgt_dev *tgt_dev,
const uint8_t *sense, int sense_len, int flags);
static void scst_free_all_UA(struct scst_tgt_dev *tgt_dev);
static void scst_release_space(struct scst_cmd *cmd);
static void scst_unblock_cmds(struct scst_device *dev);
static void scst_clear_reservation(struct scst_tgt_dev *tgt_dev);
static int scst_alloc_add_tgt_dev(struct scst_session *sess,
struct scst_acg_dev *acg_dev, struct scst_tgt_dev **out_tgt_dev);
@@ -2564,6 +2565,21 @@ void scst_free_tgt(struct scst_tgt *tgt)
return;
}
static void scst_init_order_data(struct scst_order_data *order_data)
{
int i;
spin_lock_init(&order_data->sn_lock);
INIT_LIST_HEAD(&order_data->deferred_cmd_list);
INIT_LIST_HEAD(&order_data->skipped_sn_list);
order_data->curr_sn = (typeof(order_data->curr_sn))(-300);
order_data->expected_sn = order_data->curr_sn + 1;
order_data->num_free_sn_slots = ARRAY_SIZE(order_data->sn_slots)-1;
order_data->cur_sn_slot = &order_data->sn_slots[0];
for (i = 0; i < (int)ARRAY_SIZE(order_data->sn_slots); i++)
atomic_set(&order_data->sn_slots[i], 0);
return;
}
/* Called under scst_mutex and suspended activity */
int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev)
{
@@ -2590,7 +2606,6 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev)
dev->queue_alg = SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER;
mutex_init(&dev->dev_pr_mutex);
atomic_set(&dev->pr_readers_count, 0);
dev->pr_generation = 0;
dev->pr_is_set = 0;
dev->pr_holder = NULL;
@@ -2598,6 +2613,8 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev)
dev->pr_type = TYPE_UNSPECIFIED;
INIT_LIST_HEAD(&dev->dev_registrants_list);
scst_init_order_data(&dev->dev_order_data);
scst_init_threads(&dev->dev_cmd_threads);
*out_dev = dev;
@@ -3279,7 +3296,7 @@ static int scst_alloc_add_tgt_dev(struct scst_session *sess,
struct scst_tgt_dev *tgt_dev;
struct scst_device *dev = acg_dev->dev;
struct list_head *head;
int i, sl;
int sl;
uint8_t sense_buffer[SCST_STANDARD_SENSE_LEN];
TRACE_ENTRY();
@@ -3325,15 +3342,12 @@ static int scst_alloc_add_tgt_dev(struct scst_session *sess,
INIT_LIST_HEAD(&tgt_dev->UA_list);
spin_lock_init(&tgt_dev->thr_data_lock);
INIT_LIST_HEAD(&tgt_dev->thr_data_list);
spin_lock_init(&tgt_dev->sn_lock);
INIT_LIST_HEAD(&tgt_dev->deferred_cmd_list);
INIT_LIST_HEAD(&tgt_dev->skipped_sn_list);
tgt_dev->curr_sn = (typeof(tgt_dev->curr_sn))(-300);
tgt_dev->expected_sn = tgt_dev->curr_sn + 1;
tgt_dev->num_free_sn_slots = ARRAY_SIZE(tgt_dev->sn_slots)-1;
tgt_dev->cur_sn_slot = &tgt_dev->sn_slots[0];
for (i = 0; i < (int)ARRAY_SIZE(tgt_dev->sn_slots); i++)
atomic_set(&tgt_dev->sn_slots[i], 0);
scst_init_order_data(&tgt_dev->tgt_dev_order_data);
if (dev->tst == SCST_CONTR_MODE_SEP_TASK_SETS)
tgt_dev->curr_order_data = &tgt_dev->tgt_dev_order_data;
else
tgt_dev->curr_order_data = &dev->dev_order_data;
if (dev->handler->parse_atomic &&
dev->handler->alloc_data_buf_atomic &&
@@ -4208,7 +4222,8 @@ void scst_free_cmd(struct scst_cmd *cmd)
if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
TRACE_MGMT_DBG("Freeing aborted cmd %p", cmd);
sBUG_ON(cmd->unblock_dev);
EXTRACHECKS_BUG_ON(cmd->unblock_dev || cmd->dec_on_dev_needed ||
cmd->dec_pr_readers_count_needed);
/*
* Target driver can already free sg buffer before calling
@@ -4253,8 +4268,8 @@ void scst_free_cmd(struct scst_cmd *cmd)
"%d, target %s, LUN %lld, sn %d, expected_sn %d)",
cmd, cmd->cdb[0], cmd->tgtt->name,
(long long unsigned int)cmd->lun,
cmd->sn, cmd->tgt_dev->expected_sn);
scst_unblock_deferred(cmd->tgt_dev, cmd);
cmd->sn, cmd->cur_order_data->expected_sn);
scst_unblock_deferred(cmd->cur_order_data, cmd);
}
#endif
@@ -6327,7 +6342,7 @@ void __scst_dev_check_set_UA(struct scst_device *dev,
{
TRACE_ENTRY();
TRACE_MGMT_DBG("Processing UA dev %p", dev);
TRACE_MGMT_DBG("Processing UA dev %s", dev->virt_name);
/* Check for reset UA */
if (scst_analyze_sense(sense, sense_len, SCST_SENSE_ASC_VALID,
@@ -6364,25 +6379,25 @@ static void scst_free_all_UA(struct scst_tgt_dev *tgt_dev)
}
/* No locks */
struct scst_cmd *__scst_check_deferred_commands(struct scst_tgt_dev *tgt_dev)
struct scst_cmd *__scst_check_deferred_commands(struct scst_order_data *order_data)
{
struct scst_cmd *res = NULL, *cmd, *t;
typeof(tgt_dev->expected_sn) expected_sn = tgt_dev->expected_sn;
typeof(order_data->expected_sn) expected_sn = order_data->expected_sn;
spin_lock_irq(&tgt_dev->sn_lock);
spin_lock_irq(&order_data->sn_lock);
if (unlikely(tgt_dev->hq_cmd_count != 0))
if (unlikely(order_data->hq_cmd_count != 0))
goto out_unlock;
restart:
list_for_each_entry_safe(cmd, t, &tgt_dev->deferred_cmd_list,
list_for_each_entry_safe(cmd, t, &order_data->deferred_cmd_list,
sn_cmd_list_entry) {
EXTRACHECKS_BUG_ON(cmd->queue_type ==
SCST_CMD_QUEUE_HEAD_OF_QUEUE);
if (cmd->sn == expected_sn) {
TRACE_SN("Deferred command %p (sn %d, set %d) found",
cmd, cmd->sn, cmd->sn_set);
tgt_dev->def_cmd_count--;
order_data->def_cmd_count--;
list_del(&cmd->sn_cmd_list_entry);
if (res == NULL)
res = cmd;
@@ -6400,7 +6415,7 @@ restart:
if (res != NULL)
goto out_unlock;
list_for_each_entry(cmd, &tgt_dev->skipped_sn_list,
list_for_each_entry(cmd, &order_data->skipped_sn_list,
sn_cmd_list_entry) {
EXTRACHECKS_BUG_ON(cmd->queue_type ==
SCST_CMD_QUEUE_HEAD_OF_QUEUE);
@@ -6415,21 +6430,21 @@ restart:
cmd,
(long long unsigned int)cmd->tag,
cmd->sn);
tgt_dev->def_cmd_count--;
order_data->def_cmd_count--;
list_del(&cmd->sn_cmd_list_entry);
spin_unlock_irq(&tgt_dev->sn_lock);
spin_unlock_irq(&order_data->sn_lock);
if (test_and_set_bit(SCST_CMD_CAN_BE_DESTROYED,
&cmd->cmd_flags))
scst_destroy_put_cmd(cmd);
scst_inc_expected_sn(tgt_dev, slot);
expected_sn = tgt_dev->expected_sn;
spin_lock_irq(&tgt_dev->sn_lock);
scst_inc_expected_sn(order_data, slot);
expected_sn = order_data->expected_sn;
spin_lock_irq(&order_data->sn_lock);
goto restart;
}
}
out_unlock:
spin_unlock_irq(&tgt_dev->sn_lock);
spin_unlock_irq(&order_data->sn_lock);
return res;
}
@@ -6565,26 +6580,55 @@ bool scst_del_thr_data(struct scst_tgt_dev *tgt_dev, struct task_struct *tsk)
return res;
}
static void __scst_unblock_deferred(struct scst_order_data *order_data,
struct scst_cmd *out_of_sn_cmd)
{
EXTRACHECKS_BUG_ON(!out_of_sn_cmd->sn_set);
if (out_of_sn_cmd->sn == order_data->expected_sn) {
scst_inc_expected_sn(order_data, out_of_sn_cmd->sn_slot);
scst_make_deferred_commands_active(order_data);
} else {
out_of_sn_cmd->out_of_sn = 1;
spin_lock_irq(&order_data->sn_lock);
order_data->def_cmd_count++;
list_add_tail(&out_of_sn_cmd->sn_cmd_list_entry,
&order_data->skipped_sn_list);
TRACE_SN("out_of_sn_cmd %p with sn %d added to skipped_sn_list"
" (expected_sn %d)", out_of_sn_cmd, out_of_sn_cmd->sn,
order_data->expected_sn);
spin_unlock_irq(&order_data->sn_lock);
}
return;
}
void scst_unblock_deferred(struct scst_order_data *order_data,
struct scst_cmd *out_of_sn_cmd)
{
TRACE_ENTRY();
if (!out_of_sn_cmd->sn_set) {
TRACE_SN("cmd %p without sn", out_of_sn_cmd);
goto out;
}
__scst_unblock_deferred(order_data, out_of_sn_cmd);
out:
TRACE_EXIT();
return;
}
/* dev_lock supposed to be held and BH disabled */
void scst_block_dev(struct scst_device *dev)
{
dev->block_count++;
TRACE_MGMT_DBG("Device BLOCK(new %d), dev %p", dev->block_count, dev);
TRACE_MGMT_DBG("Device BLOCK (new count %d), dev %s", dev->block_count,
dev->virt_name);
}
/* No locks */
void scst_unblock_dev(struct scst_device *dev)
{
spin_lock_bh(&dev->dev_lock);
TRACE_MGMT_DBG("Device UNBLOCK(new %d), dev %p",
dev->block_count-1, dev);
if (--dev->block_count == 0)
scst_unblock_cmds(dev);
spin_unlock_bh(&dev->dev_lock);
sBUG_ON(dev->block_count < 0);
}
/* No locks */
/* dev_lock supposed to be held and BH disabled */
bool __scst_check_blocked_dev(struct scst_cmd *cmd)
{
int res = false;
@@ -6602,144 +6646,123 @@ bool __scst_check_blocked_dev(struct scst_cmd *cmd)
goto out;
}
repeat:
if (dev->block_count > 0) {
spin_lock_bh(&dev->dev_lock);
if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
goto out_unlock;
if (dev->block_count > 0) {
TRACE_MGMT_DBG("Delaying cmd %p due to blocking "
"(tag %llu, dev %p)", cmd,
(long long unsigned int)cmd->tag, dev);
list_add_tail(&cmd->blocked_cmd_list_entry,
&dev->blocked_cmd_list);
res = true;
spin_unlock_bh(&dev->dev_lock);
goto out;
} else {
TRACE_MGMT_DBG("%s", "Somebody unblocked the device, "
"continuing");
}
spin_unlock_bh(&dev->dev_lock);
}
if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
goto out;
if (dev->dev_double_ua_possible) {
spin_lock_bh(&dev->dev_lock);
if (dev->block_count == 0) {
TRACE_MGMT_DBG("cmd %p (tag %llu), blocking further "
"cmds due to possible double reset UA (dev %p)",
cmd, (long long unsigned int)cmd->tag, dev);
scst_block_dev(dev);
if (dev->block_count > 0) {
TRACE_MGMT_DBG("Delaying cmd %p due to blocking "
"(tag %llu, op %x, dev %s)", cmd,
(long long unsigned int)cmd->tag, cmd->cdb[0],
dev->virt_name);
goto out_block;
} else if (scst_is_strictly_serialized_cmd(cmd)) {
TRACE_MGMT_DBG("cmd %p (tag %llu, op %x): blocking further "
"cmds on dev %s due to strict serialization", cmd,
(long long unsigned int)cmd->tag, cmd->cdb[0],
dev->virt_name);
scst_block_dev(dev);
if (dev->on_dev_cmd_count > 1) {
TRACE_MGMT_DBG("Delaying strictly serialized cmd %p "
"(dev %s, on_dev_cmds to wait %d)", cmd,
dev->virt_name, dev->on_dev_cmd_count-1);
EXTRACHECKS_BUG_ON(dev->strictly_serialized_cmd_waiting);
dev->strictly_serialized_cmd_waiting = 1;
goto out_block;
} else
cmd->unblock_dev = 1;
} else {
spin_unlock_bh(&dev->dev_lock);
TRACE_MGMT_DBG("Somebody blocked the device, "
"repeating (count %d)", dev->block_count);
goto repeat;
}
spin_unlock_bh(&dev->dev_lock);
}
} else if ((dev->dev_double_ua_possible) || scst_is_serialized_cmd(cmd)) {
TRACE_MGMT_DBG("cmd %p (tag %llu, op %x): blocking further cmds "
"on dev %s due to %s", cmd, (long long unsigned int)cmd->tag,
cmd->cdb[0], dev->virt_name,
dev->dev_double_ua_possible ? "possible double reset UA" :
"serialized cmd");
scst_block_dev(dev);
cmd->unblock_dev = 1;
} else
TRACE_MGMT_DBG("No blocks for device %s", dev->virt_name);
out:
TRACE_EXIT_RES(res);
return res;
out_unlock:
spin_unlock_bh(&dev->dev_lock);
out_block:
if (cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE)
list_add(&cmd->blocked_cmd_list_entry,
&dev->blocked_cmd_list);
else
list_add_tail(&cmd->blocked_cmd_list_entry,
&dev->blocked_cmd_list);
res = true;
goto out;
}
/* Called under dev_lock */
static void scst_unblock_cmds(struct scst_device *dev)
{
struct scst_cmd *cmd, *tcmd;
unsigned long flags;
TRACE_ENTRY();
local_irq_save(flags);
list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list,
blocked_cmd_list_entry) {
list_del(&cmd->blocked_cmd_list_entry);
TRACE_MGMT_DBG("Adding blocked cmd %p to active cmd list", cmd);
spin_lock(&cmd->cmd_threads->cmd_list_lock);
if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
list_add(&cmd->cmd_list_entry,
&cmd->cmd_threads->active_cmd_list);
else
list_add_tail(&cmd->cmd_list_entry,
&cmd->cmd_threads->active_cmd_list);
wake_up(&cmd->cmd_threads->cmd_list_waitQ);
spin_unlock(&cmd->cmd_threads->cmd_list_lock);
}
local_irq_restore(flags);
TRACE_EXIT();
return;
}
static void __scst_unblock_deferred(struct scst_tgt_dev *tgt_dev,
struct scst_cmd *out_of_sn_cmd)
{
EXTRACHECKS_BUG_ON(!out_of_sn_cmd->sn_set);
if (out_of_sn_cmd->sn == tgt_dev->expected_sn) {
scst_inc_expected_sn(tgt_dev, out_of_sn_cmd->sn_slot);
scst_make_deferred_commands_active(tgt_dev);
} else {
out_of_sn_cmd->out_of_sn = 1;
spin_lock_irq(&tgt_dev->sn_lock);
tgt_dev->def_cmd_count++;
list_add_tail(&out_of_sn_cmd->sn_cmd_list_entry,
&tgt_dev->skipped_sn_list);
TRACE_SN("out_of_sn_cmd %p with sn %d added to skipped_sn_list"
" (expected_sn %d)", out_of_sn_cmd, out_of_sn_cmd->sn,
tgt_dev->expected_sn);
spin_unlock_irq(&tgt_dev->sn_lock);
}
return;
}
void scst_unblock_deferred(struct scst_tgt_dev *tgt_dev,
struct scst_cmd *out_of_sn_cmd)
/* dev_lock supposed to be held and BH disabled */
void scst_unblock_dev(struct scst_device *dev)
{
TRACE_ENTRY();
if (!out_of_sn_cmd->sn_set) {
TRACE_SN("cmd %p without sn", out_of_sn_cmd);
goto out;
TRACE_MGMT_DBG("Device UNBLOCK(new %d), dev %s",
dev->block_count-1, dev->virt_name);
EXTRACHECKS_BUG_ON(!spin_is_locked(&dev->dev_lock));
if (--dev->block_count == 0) {
struct scst_cmd *cmd, *tcmd;
unsigned long flags;
local_irq_save(flags);
list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list,
blocked_cmd_list_entry) {
bool strictly_serialized;
list_del(&cmd->blocked_cmd_list_entry);
TRACE_MGMT_DBG("Adding blocked cmd %p to active cmd "
"list", cmd);
spin_lock(&cmd->cmd_threads->cmd_list_lock);
if (cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE)
list_add(&cmd->cmd_list_entry,
&cmd->cmd_threads->active_cmd_list);
else
list_add_tail(&cmd->cmd_list_entry,
&cmd->cmd_threads->active_cmd_list);
strictly_serialized = scst_is_strictly_serialized_cmd(cmd);
wake_up(&cmd->cmd_threads->cmd_list_waitQ);
spin_unlock(&cmd->cmd_threads->cmd_list_lock);
if (dev->strictly_serialized_cmd_waiting && strictly_serialized)
break;
}
local_irq_restore(flags);
dev->strictly_serialized_cmd_waiting = 0;
}
__scst_unblock_deferred(tgt_dev, out_of_sn_cmd);
sBUG_ON(dev->block_count < 0);
out:
TRACE_EXIT();
return;
}
void scst_on_hq_cmd_response(struct scst_cmd *cmd)
{
struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
struct scst_order_data *order_data = cmd->cur_order_data;
TRACE_ENTRY();
if (!cmd->hq_cmd_inced)
goto out;
spin_lock_irq(&tgt_dev->sn_lock);
tgt_dev->hq_cmd_count--;
spin_unlock_irq(&tgt_dev->sn_lock);
spin_lock_irq(&order_data->sn_lock);
order_data->hq_cmd_count--;
spin_unlock_irq(&order_data->sn_lock);
EXTRACHECKS_BUG_ON(tgt_dev->hq_cmd_count < 0);
EXTRACHECKS_BUG_ON(order_data->hq_cmd_count < 0);
/*
* There is no problem in checking hq_cmd_count in the
* non-locked state. In the worst case we will only have
* unneeded run of the deferred commands.
*/
if (tgt_dev->hq_cmd_count == 0)
scst_make_deferred_commands_active(tgt_dev);
if (order_data->hq_cmd_count == 0)
scst_make_deferred_commands_active(order_data);
out:
TRACE_EXIT();

10
scst/src/scst_main.c
View File

@@ -1571,8 +1571,8 @@ int scst_add_threads(struct scst_cmd_threads *cmd_threads,
n++;
}
TRACE_DBG("cmd_threads %p, dev %p, tgt_dev %p, num %d, n %d",
cmd_threads, dev, tgt_dev, num, n);
TRACE_DBG("cmd_threads %p, dev %s, tgt_dev %p, num %d, n %d",
cmd_threads, dev->virt_name, tgt_dev, num, n);
if (tgt_dev != NULL) {
struct scst_tgt_dev *t;
@@ -2139,11 +2139,11 @@ static int __init init_scst(void)
{
struct scsi_sense_hdr *shdr;
struct scst_tgt_dev *t;
struct scst_order_data *o;
struct scst_cmd *c;
BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
BUILD_BUG_ON(sizeof(o->curr_sn) != sizeof(o->expected_sn));
BUILD_BUG_ON(sizeof(c->sn) != sizeof(o->expected_sn));
}
mutex_init(&scst_mutex);

6
scst/src/scst_pres.c
View File

@@ -2458,7 +2458,7 @@ bool scst_pr_is_cmd_allowed(struct scst_cmd *cmd)
TRACE_ENTRY();
unlock = scst_pr_read_lock(dev);
unlock = scst_pr_read_lock(cmd);
TRACE_DBG("Testing if command %s (0x%x) from %s allowed to execute",
cmd->op_name, cmd->cdb[0], cmd->sess->initiator_name);
@@ -2518,7 +2518,7 @@ bool scst_pr_is_cmd_allowed(struct scst_cmd *cmd)
cmd->op_name, cmd->cdb[0], cmd->sess->initiator_name);
out_unlock:
scst_pr_read_unlock(dev, unlock);
scst_pr_read_unlock(cmd, unlock);
TRACE_EXIT_RES(allowed);
return allowed;
@@ -2540,7 +2540,7 @@ void scst_pr_read_keys(struct scst_cmd *cmd, uint8_t *buffer, int buffer_size)
}
TRACE_PR("Read Keys (dev %s): PRGen %d", dev->virt_name,
dev->pr_generation);
dev->pr_generation);
put_unaligned(cpu_to_be32(dev->pr_generation), (__be32 *)&buffer[0]);

82
scst/src/scst_pres.h
View File

@@ -45,6 +45,52 @@
#define SCOPE_LU 0x00
static inline void scst_inc_pr_readers_count(struct scst_cmd *cmd,
bool locked)
{
struct scst_device *dev = cmd->dev;
EXTRACHECKS_BUG_ON(cmd->dec_pr_readers_count_needed);
if (!locked)
spin_lock_bh(&dev->dev_lock);
EXTRACHECKS_BUG_ON(!spin_is_locked(&dev->dev_lock));
dev->pr_readers_count++;
cmd->dec_pr_readers_count_needed = 1;
TRACE_DBG("New inc pr_readers_count %d (cmd %p)", dev->pr_readers_count,
cmd);
if (!locked)
spin_unlock_bh(&dev->dev_lock);
return;
}
static inline void scst_dec_pr_readers_count(struct scst_cmd *cmd,
bool locked)
{
struct scst_device *dev = cmd->dev;
EXTRACHECKS_BUG_ON(!cmd->dec_pr_readers_count_needed);
if (!locked)
spin_lock_bh(&dev->dev_lock);
EXTRACHECKS_BUG_ON(!spin_is_locked(&dev->dev_lock));
dev->pr_readers_count--;
cmd->dec_pr_readers_count_needed = 0;
TRACE_DBG("New dec pr_readers_count %d (cmd %p)", dev->pr_readers_count,
cmd);
if (!locked)
spin_unlock_bh(&dev->dev_lock);
EXTRACHECKS_BUG_ON(dev->pr_readers_count < 0);
return;
}
static inline bool scst_pr_type_valid(uint8_t type)
{
switch (type) {
@@ -60,18 +106,17 @@ static inline bool scst_pr_type_valid(uint8_t type)
}
}
static inline bool scst_pr_read_lock(struct scst_device *dev)
static inline bool scst_pr_read_lock(struct scst_cmd *cmd)
{
struct scst_device *dev = cmd->dev;
bool unlock = false;
TRACE_ENTRY();
atomic_inc(&dev->pr_readers_count);
smp_mb__after_atomic_inc(); /* to sync with scst_pr_write_lock() */
smp_mb(); /* to sync with scst_pr_write_lock() */
if (unlikely(dev->pr_writer_active)) {
unlock = true;
atomic_dec(&dev->pr_readers_count);
scst_dec_pr_readers_count(cmd, false);
mutex_lock(&dev->dev_pr_mutex);
}
@@ -79,20 +124,16 @@ static inline bool scst_pr_read_lock(struct scst_device *dev)
return unlock;
}
static inline void scst_pr_read_unlock(struct scst_device *dev, bool unlock)
static inline void scst_pr_read_unlock(struct scst_cmd *cmd, bool unlock)
{
struct scst_device *dev = cmd->dev;
TRACE_ENTRY();
if (unlikely(unlock))
mutex_unlock(&dev->dev_pr_mutex);
else {
/*
* To sync with scst_pr_write_lock(). We need it to ensure
* order of our reads with the writer's writes.
*/
smp_mb__before_atomic_dec();
atomic_dec(&dev->pr_readers_count);
}
else
scst_dec_pr_readers_count(cmd, false);
TRACE_EXIT();
return;
@@ -105,13 +146,17 @@ static inline void scst_pr_write_lock(struct scst_device *dev)
mutex_lock(&dev->dev_pr_mutex);
dev->pr_writer_active = 1;
/* to sync with scst_pr_read_lock() and unlock() */
smp_mb();
while (atomic_read(&dev->pr_readers_count) != 0) {
TRACE_DBG("Waiting for %d readers (dev %p)",
atomic_read(&dev->pr_readers_count), dev);
while (true) {
int readers;
spin_lock_bh(&dev->dev_lock);
readers = dev->pr_readers_count;
spin_unlock_bh(&dev->dev_lock);
if (readers == 0)
break;
TRACE_DBG("Waiting for %d readers (dev %p)", readers, dev);
msleep(1);
}
@@ -124,7 +169,6 @@ static inline void scst_pr_write_unlock(struct scst_device *dev)
TRACE_ENTRY();
dev->pr_writer_active = 0;
mutex_unlock(&dev->dev_pr_mutex);
TRACE_EXIT();

55
scst/src/scst_priv.h
View File

@@ -251,24 +251,24 @@ extern bool scst_del_thr_data(struct scst_tgt_dev *tgt_dev,
extern struct scst_dev_type scst_null_devtype;
extern struct scst_cmd *__scst_check_deferred_commands(
struct scst_tgt_dev *tgt_dev);
struct scst_order_data *order_data);
/* Used to save the function call on the fast path */
static inline struct scst_cmd *scst_check_deferred_commands(
struct scst_tgt_dev *tgt_dev)
struct scst_order_data *order_data)
{
if (tgt_dev->def_cmd_count == 0)
if (order_data->def_cmd_count == 0)
return NULL;
else
return __scst_check_deferred_commands(tgt_dev);
return __scst_check_deferred_commands(order_data);
}
static inline void scst_make_deferred_commands_active(
struct scst_tgt_dev *tgt_dev)
struct scst_order_data *order_data)
{
struct scst_cmd *c;
c = __scst_check_deferred_commands(tgt_dev);
c = __scst_check_deferred_commands(order_data);
if (c != NULL) {
TRACE_SN("Adding cmd %p to active cmd list", c);
spin_lock_irq(&c->cmd_threads->cmd_list_lock);
@@ -281,10 +281,10 @@ static inline void scst_make_deferred_commands_active(
return;
}
void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot);
void scst_inc_expected_sn(struct scst_order_data *order_data, atomic_t *slot);
int scst_check_hq_cmd(struct scst_cmd *cmd);
void scst_unblock_deferred(struct scst_tgt_dev *tgt_dev,
void scst_unblock_deferred(struct scst_order_data *order_data,
struct scst_cmd *cmd_sn);
void scst_on_hq_cmd_response(struct scst_cmd *cmd);
@@ -537,43 +537,32 @@ void scst_free_aen(struct scst_aen *aen);
void scst_gen_aen_or_ua(struct scst_tgt_dev *tgt_dev,
int key, int asc, int ascq);
static inline bool scst_is_implicit_hq(struct scst_cmd *cmd)
static inline bool scst_is_implicit_hq_cmd(struct scst_cmd *cmd)
{
return (cmd->op_flags & SCST_IMPLICIT_HQ) != 0;
}
static inline bool scst_is_serialized_cmd(struct scst_cmd *cmd)
{
return (cmd->op_flags & SCST_SERIALIZED) != 0;
}
static inline bool scst_is_strictly_serialized_cmd(struct scst_cmd *cmd)
{
return (cmd->op_flags & SCST_STRICTLY_SERIALIZED) == SCST_STRICTLY_SERIALIZED;
}
/*
* Some notes on devices "blocking". Blocking means that no
* commands will go from SCST to underlying SCSI device until it
* is unblocked. But we don't care about all commands that
* already on the device.
* is unblocked. But, except for strictly serialized commands,
* we don't care about all commands that already on the device.
*/
extern void scst_block_dev(struct scst_device *dev);
extern void scst_unblock_dev(struct scst_device *dev);
extern bool __scst_check_blocked_dev(struct scst_cmd *cmd);
static inline bool scst_check_blocked_dev(struct scst_cmd *cmd)
{
if (unlikely(cmd->dev->block_count > 0) ||
unlikely(cmd->dev->dev_double_ua_possible))
return __scst_check_blocked_dev(cmd);
else
return false;
}
/* No locks */
static inline void scst_check_unblock_dev(struct scst_cmd *cmd)
{
if (unlikely(cmd->unblock_dev)) {
TRACE_MGMT_DBG("cmd %p (tag %llu): unblocking dev %p", cmd,
(long long unsigned int)cmd->tag, cmd->dev);
cmd->unblock_dev = 0;
scst_unblock_dev(cmd->dev);
}
return;
}
bool __scst_check_blocked_dev(struct scst_cmd *cmd);
/*
* Increases global SCST ref counters which prevent from entering into suspended

252
scst/src/scst_targ.c
View File

@@ -102,6 +102,77 @@ static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
return;
}
/* No locks */
static bool scst_check_blocked_dev(struct scst_cmd *cmd)
{
bool res;
struct scst_device *dev = cmd->dev;
spin_lock_bh(&dev->dev_lock);
dev->on_dev_cmd_count++;
cmd->dec_on_dev_needed = 1;
TRACE_DBG("New inc on_dev_count %d (cmd %p)", dev->on_dev_cmd_count,
cmd);
scst_inc_pr_readers_count(cmd, true);
if (unlikely(dev->block_count > 0) ||
unlikely(dev->dev_double_ua_possible) ||
unlikely(scst_is_serialized_cmd(cmd)))
res = __scst_check_blocked_dev(cmd);
else
res = false;
if (unlikely(res)) {
/* Undo increments */
dev->on_dev_cmd_count--;
cmd->dec_on_dev_needed = 0;
TRACE_DBG("New dec on_dev_count %d (cmd %p)",
dev->on_dev_cmd_count, cmd);
scst_dec_pr_readers_count(cmd, true);
}
spin_unlock_bh(&dev->dev_lock);
return res;
}
/* No locks */
static void scst_check_unblock_dev(struct scst_cmd *cmd)
{
struct scst_device *dev = cmd->dev;
spin_lock_bh(&dev->dev_lock);
if (likely(cmd->dec_on_dev_needed)) {
dev->on_dev_cmd_count--;
cmd->dec_on_dev_needed = 0;
TRACE_DBG("New dec on_dev_count %d (cmd %p)",
dev->on_dev_cmd_count, cmd);
}
if (unlikely(cmd->dec_pr_readers_count_needed))
scst_dec_pr_readers_count(cmd, true);
if (unlikely(cmd->unblock_dev)) {
TRACE_MGMT_DBG("cmd %p (tag %llu): unblocking dev %s", cmd,
(long long unsigned int)cmd->tag, dev->virt_name);
cmd->unblock_dev = 0;
scst_unblock_dev(dev);
} else if (unlikely(dev->strictly_serialized_cmd_waiting)) {
if (dev->on_dev_cmd_count == 0) {
TRACE_MGMT_DBG("Strictly serialized cmd waiting: "
"unblocking dev %s", dev->virt_name);
scst_unblock_dev(dev);
}
}
spin_unlock_bh(&dev->dev_lock);
return;
}
/**
* scst_rx_cmd() - create new command
* @sess: SCST session
@@ -2304,7 +2375,7 @@ out_done:
* !! Dev handlers implementing exec() callback must call this function there
* !! just before the actual command's execution!
*
* On call no locks, no IRQ or IRQ-disabled context allowed.
* On call no locks, no IRQ or IRQ-disabled context allowed.
*/
int scst_check_local_events(struct scst_cmd *cmd)
{
@@ -2327,7 +2398,7 @@ int scst_check_local_events(struct scst_cmd *cmd)
if ((cmd->op_flags & SCST_REG_RESERVE_ALLOWED) == 0) {
scst_set_cmd_error_status(cmd,
SAM_STAT_RESERVATION_CONFLICT);
goto out_complete;
goto out_dec_pr_readers_count;
}
}
@@ -2337,7 +2408,8 @@ int scst_check_local_events(struct scst_cmd *cmd)
SAM_STAT_RESERVATION_CONFLICT);
goto out_complete;
}
}
} else
scst_dec_pr_readers_count(cmd, false);
/*
* Let's check for ABORTED after scst_pr_is_cmd_allowed(), because
@@ -2354,8 +2426,7 @@ int scst_check_local_events(struct scst_cmd *cmd)
if (scst_is_ua_command(cmd)) {
int done = 0;
/*
* Prevent more than 1 cmd to be triggered by
* was_reset.
* Prevent more than 1 cmd to be triggered by was_reset
*/
spin_lock_bh(&dev->dev_lock);
if (dev->scsi_dev->was_reset) {
@@ -2364,7 +2435,7 @@ int scst_check_local_events(struct scst_cmd *cmd)
SCST_LOAD_SENSE(scst_sense_reset_UA));
/*
* It looks like it is safe to clear was_reset
* here.
* here
*/
dev->scsi_dev->was_reset = 0;
done = 1;
@@ -2391,6 +2462,10 @@ out:
TRACE_EXIT_RES(res);
return res;
out_dec_pr_readers_count:
if (cmd->dec_pr_readers_count_needed)
scst_dec_pr_readers_count(cmd, false);
out_complete:
res = 1;
sBUG_ON(!cmd->completed);
@@ -2403,36 +2478,34 @@ out_uncomplete:
EXPORT_SYMBOL_GPL(scst_check_local_events);
/* No locks */
void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
void scst_inc_expected_sn(struct scst_order_data *order_data, atomic_t *slot)
{
if (slot == NULL)
goto inc;
/* Optimized for lockless fast path */
TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - order_data->sn_slots,
atomic_read(slot));
if (!atomic_dec_and_test(slot))
goto out;
TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
tgt_dev->num_free_sn_slots);
if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
spin_lock_irq(&tgt_dev->sn_lock);
if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
if (tgt_dev->num_free_sn_slots < 0)
tgt_dev->cur_sn_slot = slot;
/*
* To be in-sync with SIMPLE case in scst_cmd_set_sn()
*/
order_data->num_free_sn_slots);
if (order_data->num_free_sn_slots < (int)ARRAY_SIZE(order_data->sn_slots)-1) {
spin_lock_irq(&order_data->sn_lock);
if (likely(order_data->num_free_sn_slots < (int)ARRAY_SIZE(order_data->sn_slots)-1)) {
if (order_data->num_free_sn_slots < 0)
order_data->cur_sn_slot = slot;
/* To be in-sync with SIMPLE case in scst_cmd_set_sn() */
smp_mb();
tgt_dev->num_free_sn_slots++;
order_data->num_free_sn_slots++;
TRACE_SN("Incremented num_free_sn_slots (%d)",
tgt_dev->num_free_sn_slots);
order_data->num_free_sn_slots);
}
spin_unlock_irq(&tgt_dev->sn_lock);
spin_unlock_irq(&order_data->sn_lock);
}
inc:
@@ -2441,13 +2514,13 @@ inc:
* at time can be here (serialized by sn). Also it is supposed that
* there could not be half-incremented halves.
*/
tgt_dev->expected_sn++;
order_data->expected_sn++;
/*
* Write must be before def_cmd_count read to be in sync. with
* scst_post_exec_sn(). See comment in scst_send_for_exec().
*/
smp_mb();
TRACE_SN("Next expected_sn: %d", tgt_dev->expected_sn);
TRACE_SN("Next expected_sn: %d", order_data->expected_sn);
out:
return;
@@ -2460,19 +2533,19 @@ static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
/* For HQ commands SN is not set */
bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
cmd->sn_set && !cmd->retry;
struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
struct scst_order_data *order_data = cmd->cur_order_data;
struct scst_cmd *res;
TRACE_ENTRY();
if (inc_expected_sn)
scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
scst_inc_expected_sn(order_data, cmd->sn_slot);
if (make_active) {
scst_make_deferred_commands_active(tgt_dev);
scst_make_deferred_commands_active(order_data);
res = NULL;
} else
res = scst_check_deferred_commands(tgt_dev);
res = scst_check_deferred_commands(order_data);
TRACE_EXIT_HRES(res);
return res;
@@ -2766,8 +2839,8 @@ static int scst_send_for_exec(struct scst_cmd **active_cmd)
{
int res;
struct scst_cmd *cmd = *active_cmd;
struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
typeof(tgt_dev->expected_sn) expected_sn;
struct scst_order_data *order_data = cmd->cur_order_data;
typeof(order_data->expected_sn) expected_sn;
TRACE_ENTRY();
@@ -2779,12 +2852,12 @@ static int scst_send_for_exec(struct scst_cmd **active_cmd)
sBUG_ON(!cmd->sn_set);
expected_sn = tgt_dev->expected_sn;
expected_sn = order_data->expected_sn;
/* Optimized for lockless fast path */
if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
spin_lock_irq(&tgt_dev->sn_lock);
if ((cmd->sn != expected_sn) || (order_data->hq_cmd_count > 0)) {
spin_lock_irq(&order_data->sn_lock);
tgt_dev->def_cmd_count++;
order_data->def_cmd_count++;
/*
* Memory barrier is needed here to implement lockless fast
* path. We need the exact order of read and write between
@@ -2798,15 +2871,15 @@ static int scst_send_for_exec(struct scst_cmd **active_cmd)
*/
smp_mb();
expected_sn = tgt_dev->expected_sn;
if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
expected_sn = order_data->expected_sn;
if ((cmd->sn != expected_sn) || (order_data->hq_cmd_count > 0)) {
if (unlikely(test_bit(SCST_CMD_ABORTED,
&cmd->cmd_flags))) {
/* Necessary to allow aborting out of sn cmds */
TRACE_MGMT_DBG("Aborting out of sn cmd %p "
"(tag %llu, sn %u)", cmd,
(long long unsigned)cmd->tag, cmd->sn);
tgt_dev->def_cmd_count--;
order_data->def_cmd_count--;
scst_set_cmd_abnormal_done_state(cmd);
res = SCST_CMD_STATE_RES_CONT_SAME;
} else {
@@ -2814,16 +2887,16 @@ static int scst_send_for_exec(struct scst_cmd **active_cmd)
"expected_sn=%d)", cmd, cmd->sn,
cmd->sn_set, expected_sn);
list_add_tail(&cmd->sn_cmd_list_entry,
&tgt_dev->deferred_cmd_list);
&order_data->deferred_cmd_list);
res = SCST_CMD_STATE_RES_CONT_NEXT;
}
spin_unlock_irq(&tgt_dev->sn_lock);
spin_unlock_irq(&order_data->sn_lock);
goto out;
} else {
TRACE_SN("Somebody incremented expected_sn %d, "
"continuing", expected_sn);
tgt_dev->def_cmd_count--;
spin_unlock_irq(&tgt_dev->sn_lock);
order_data->def_cmd_count--;
spin_unlock_irq(&order_data->sn_lock);
}
}
@@ -3204,9 +3277,9 @@ out:
static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
{
if (likely(cmd->sn_set))
scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
scst_inc_expected_sn(cmd->cur_order_data, cmd->sn_slot);
scst_make_deferred_commands_active(cmd->tgt_dev);
scst_make_deferred_commands_active(cmd->cur_order_data);
}
static int scst_dev_done(struct scst_cmd *cmd)
@@ -3361,7 +3434,7 @@ static int scst_pre_xmit_response(struct scst_cmd *cmd)
TRACE_SN("cmd %p was not sent to mid-lev"
" (sn %d, set %d)",
cmd, cmd->sn, cmd->sn_set);
scst_unblock_deferred(cmd->tgt_dev, cmd);
scst_unblock_deferred(cmd->cur_order_data, cmd);
cmd->sent_for_exec = 1;
}
}
@@ -3621,12 +3694,12 @@ out:
*/
static void scst_cmd_set_sn(struct scst_cmd *cmd)
{
struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
struct scst_order_data *order_data = cmd->cur_order_data;
unsigned long flags;
TRACE_ENTRY();
if (scst_is_implicit_hq(cmd) &&
if (scst_is_implicit_hq_cmd(cmd) &&
likely(cmd->queue_type == SCST_CMD_QUEUE_SIMPLE)) {
TRACE_SN("Implicit HQ cmd %p", cmd);
cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
@@ -3654,23 +3727,23 @@ static void scst_cmd_set_sn(struct scst_cmd *cmd)
switch (cmd->queue_type) {
case SCST_CMD_QUEUE_SIMPLE:
case SCST_CMD_QUEUE_UNTAGGED:
if (likely(tgt_dev->num_free_sn_slots >= 0)) {
if (likely(order_data->num_free_sn_slots >= 0)) {
/*
* atomic_inc_return() implies memory barrier to sync
* with scst_inc_expected_sn()
*/
if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
tgt_dev->curr_sn++;
if (atomic_inc_return(order_data->cur_sn_slot) == 1) {
order_data->curr_sn++;
TRACE_SN("Incremented curr_sn %d",
tgt_dev->curr_sn);
order_data->curr_sn);
}
cmd->sn_slot = tgt_dev->cur_sn_slot;
cmd->sn = tgt_dev->curr_sn;
cmd->sn_slot = order_data->cur_sn_slot;
cmd->sn = order_data->curr_sn;
tgt_dev->prev_cmd_ordered = 0;
order_data->prev_cmd_ordered = 0;
} else {
TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
"%zd", ARRAY_SIZE(tgt_dev->sn_slots));
"%zd", ARRAY_SIZE(order_data->sn_slots));
goto ordered;
}
break;
@@ -3678,44 +3751,44 @@ static void scst_cmd_set_sn(struct scst_cmd *cmd)
case SCST_CMD_QUEUE_ORDERED:
TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
ordered:
if (!tgt_dev->prev_cmd_ordered) {
spin_lock_irqsave(&tgt_dev->sn_lock, flags);
if (tgt_dev->num_free_sn_slots >= 0) {
tgt_dev->num_free_sn_slots--;
if (tgt_dev->num_free_sn_slots >= 0) {
if (!order_data->prev_cmd_ordered) {
spin_lock_irqsave(&order_data->sn_lock, flags);
if (order_data->num_free_sn_slots >= 0) {
order_data->num_free_sn_slots--;
if (order_data->num_free_sn_slots >= 0) {
int i = 0;
/* Commands can finish in any order, so
* we don't know which slot is empty.
*/
while (1) {
tgt_dev->cur_sn_slot++;
if (tgt_dev->cur_sn_slot ==
tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
order_data->cur_sn_slot++;
if (order_data->cur_sn_slot ==
order_data->sn_slots + ARRAY_SIZE(order_data->sn_slots))
order_data->cur_sn_slot = order_data->sn_slots;
if (atomic_read(tgt_dev->cur_sn_slot) == 0)
if (atomic_read(order_data->cur_sn_slot) == 0)
break;
i++;
sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
sBUG_ON(i == ARRAY_SIZE(order_data->sn_slots));
}
TRACE_SN("New cur SN slot %zd",
tgt_dev->cur_sn_slot -
tgt_dev->sn_slots);
order_data->cur_sn_slot -
order_data->sn_slots);
}
}
spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
spin_unlock_irqrestore(&order_data->sn_lock, flags);
}
tgt_dev->prev_cmd_ordered = 1;
tgt_dev->curr_sn++;
cmd->sn = tgt_dev->curr_sn;
order_data->prev_cmd_ordered = 1;
order_data->curr_sn++;
cmd->sn = order_data->curr_sn;
break;
case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
spin_lock_irqsave(&tgt_dev->sn_lock, flags);
tgt_dev->hq_cmd_count++;
spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
spin_lock_irqsave(&order_data->sn_lock, flags);
order_data->hq_cmd_count++;
spin_unlock_irqrestore(&order_data->sn_lock, flags);
cmd->hq_cmd_inced = 1;
goto out;
@@ -3723,12 +3796,12 @@ ordered:
sBUG();
}
TRACE_SN("cmd(%p)->sn: %d (tgt_dev %p, *cur_sn_slot %d, "
TRACE_SN("cmd(%p)->sn: %d (order_data %p, *cur_sn_slot %d, "
"num_free_sn_slots %d, prev_cmd_ordered %ld, "
"cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
atomic_read(tgt_dev->cur_sn_slot),
tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
"cur_sn_slot %zd)", cmd, cmd->sn, order_data,
atomic_read(order_data->cur_sn_slot),
order_data->num_free_sn_slots, order_data->prev_cmd_ordered,
order_data->cur_sn_slot - order_data->sn_slots);
cmd->sn_set = 1;
@@ -3774,6 +3847,7 @@ static int scst_translate_lun(struct scst_cmd *cmd)
cmd->cmd_threads = tgt_dev->active_cmd_threads;
cmd->tgt_dev = tgt_dev;
cmd->cur_order_data = tgt_dev->curr_order_data;
cmd->dev = tgt_dev->dev;
res = 0;
@@ -4830,20 +4904,21 @@ static void scst_unblock_aborted_cmds(int scst_mutex_held)
local_irq_disable();
list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
dev_tgt_dev_list_entry) {
spin_lock(&tgt_dev->sn_lock);
dev_tgt_dev_list_entry) {
struct scst_order_data *order_data = tgt_dev->curr_order_data;
spin_lock(&order_data->sn_lock);
list_for_each_entry_safe(cmd, tcmd,
&tgt_dev->deferred_cmd_list,
&order_data->deferred_cmd_list,
sn_cmd_list_entry) {
if (__scst_check_unblock_aborted_cmd(cmd,
&cmd->sn_cmd_list_entry)) {
TRACE_MGMT_DBG("Unblocked aborted SN "
"cmd %p (sn %u)",
cmd, cmd->sn);
tgt_dev->def_cmd_count--;
order_data->def_cmd_count--;
}
}
spin_unlock(&tgt_dev->sn_lock);
spin_unlock(&order_data->sn_lock);
}
local_irq_enable();
}
@@ -4935,13 +5010,13 @@ static int scst_is_cmd_belongs_to_dev(struct scst_cmd *cmd,
TRACE_ENTRY();
TRACE_DBG("Finding match for dev %p and cmd %p (lun %lld)", dev, cmd,
(long long unsigned int)cmd->lun);
TRACE_DBG("Finding match for dev %s and cmd %p (lun %lld)", dev->virt_name,
cmd, (long long unsigned int)cmd->lun);
head = &cmd->sess->sess_tgt_dev_list[SESS_TGT_DEV_LIST_HASH_FN(cmd->lun)];
list_for_each_entry(tgt_dev, head, sess_tgt_dev_list_entry) {
if (tgt_dev->lun == cmd->lun) {
TRACE_DBG("dev %p found", tgt_dev->dev);
TRACE_DBG("dev %s found", tgt_dev->dev->virt_name);
res = (tgt_dev->dev == dev);
goto out;
}
@@ -5615,7 +5690,10 @@ static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd)
switch (mcmd->fn) {
case SCST_LUN_RESET:
case SCST_CLEAR_TASK_SET:
scst_unblock_dev(mcmd->mcmd_tgt_dev->dev);
dev = mcmd->mcmd_tgt_dev->dev;
spin_lock_bh(&dev->dev_lock);
scst_unblock_dev(dev);
spin_unlock_bh(&dev->dev_lock);
break;
case SCST_TARGET_RESET:
@@ -5627,7 +5705,9 @@ static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd)
list_for_each_entry(acg_dev, &acg->acg_dev_list,
acg_dev_list_entry) {
dev = acg_dev->dev;
spin_lock_bh(&dev->dev_lock);
scst_unblock_dev(dev);
spin_unlock_bh(&dev->dev_lock);
}
mutex_unlock(&scst_mutex);
break;