mirrors/scst
1
0
Fork 0
mirror of https://github.com/SCST-project/scst.git synced 2026-07-19 22:42:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

New COMPARE AND WRITE implementation

Browse Source 
Prepared with help from Sushma Gurram <Sushma.Gurram@sandisk.com>



git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@6588 d57e44dd-8a1f-0410-8b47-8ef2f437770f
This commit is contained in:
Vladislav Bolkhovitin
2015-11-06 03:56:48 +00:00
parent aff6012d44
commit 54cf21076a
5 changed files with 1099 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scst/include/scst.h
+38
View File
@@ -2136,6 +2136,12 @@ struct scst_cmd {
/* Set if scst_dec_on_dev_cmd() call is needed on the cmd's finish */
unsigned int dec_on_dev_needed:1;
/* Set if cmd is on dev's exec_cmd_list */
unsigned int on_dev_exec_list:1;
/* Set if this cmd passed check for SCSI atomicity */
unsigned int scsi_atomicity_checked:1;
/* Set if cmd is queued as hw pending */
unsigned int cmd_hw_pending:1;
@@ -2298,6 +2304,15 @@ struct scst_cmd {
unsigned short cdb_len;
uint8_t cdb_buf[SCST_MAX_CDB_SIZE];
/* List entry for dev's dev_exec_cmd_list */
struct list_head dev_exec_cmd_list_entry;
/*
* Array of blocked by this cmd SCSI atomic cmds with size
* scsi_atomic_blocked_cmds_count. Protected by dev->dev_lock.
*/
struct scst_cmd **scsi_atomic_blocked_cmds;
uint8_t lba_off; /* LBA offset in cdb */
uint8_t lba_len; /* LBA length in cdb */
uint8_t len_off; /* length offset in cdb */
@@ -2429,6 +2444,18 @@ struct scst_cmd {
/* Used for storage of dev handler private stuff */
void *dh_priv;
/*
* Number of waiting for this cmd to finish commands
* with SCSI atomic guarantees. Protected by dev->dev_lock.
*/
int scsi_atomic_blockers;
/*
* How many SCSI atomic cmds this cmd blocked, i.e. size of array
* scsi_atomic_blocked_cmds. Protected by dev->dev_lock.
*/
int scsi_atomic_blocked_cmds_count;
/* List entry for dev's blocked_cmd_list */
struct list_head blocked_cmd_list_entry;
@@ -2812,6 +2839,17 @@ struct scst_device {
*/
int on_dev_cmd_count;
/*
* How many atomic SCSI commands being executed. Protected by dev_lock.
*/
int dev_scsi_atomic_cmd_active;
/*
* List of all being executed on the dev commands.
* Protected by dev_lock.
*/
struct list_head dev_exec_cmd_list;
/* Memory limits for this device */
struct scst_mem_lim dev_mem_lim;
scst/include/scst_const.h
+4 -3
View File
@@ -228,9 +228,10 @@ enum scst_cdb_flags {
#ifdef CONFIG_SCST_TEST_IO_IN_SIRQ
SCST_TEST_IO_IN_SIRQ_ALLOWED = 0x8000,
#endif
SCST_SERIALIZED = 0x10000,
SCST_STRICTLY_SERIALIZED = 0x20000|SCST_SERIALIZED,
SCST_DESCRIPTORS_BASED = 0x40000,
SCST_SERIALIZED = 0x10000,
SCST_STRICTLY_SERIALIZED = 0x20000|SCST_SERIALIZED,
SCST_DESCRIPTORS_BASED = 0x40000,
SCST_SCSI_ATOMIC = 0x100000,
};
/*************************************************************
scst/src/scst_lib.c
+646 -6
View File
@@ -155,6 +155,12 @@ static int sg_copy(struct scatterlist *dst_sg, struct scatterlist *src_sg,
#endif
static void scst_free_descriptors(struct scst_cmd *cmd);
static bool sg_cmp(struct scatterlist *dst_sg, struct scatterlist *src_sg,
int nents_to_cmp, size_t cmp_len, int *miscompare_offs
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
, enum km_type d_km_type, enum km_type s_km_type
#endif
);
const struct scst_opcode_descriptor scst_op_descr_inquiry = {
.od_opcode = INQUIRY,
@@ -757,7 +763,8 @@ static const struct scst_sdbops scst_scsi_op_table[] = {
.info_op_name = "RESERVE",
.info_data_direction = SCST_DATA_NONE,
.info_op_flags = SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED|
SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED,
SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED|
SCST_SCSI_ATOMIC/* see comment in scst_cmd_overlap() */,
.get_cdb_info = get_cdb_info_none},
{.ops = 0x17, .devkey = "MMMMMMMMMMMMMMMM",
.info_op_name = "RELEASE",
@@ -1170,7 +1177,8 @@ static const struct scst_sdbops scst_scsi_op_table[] = {
.info_op_name = "RESERVE(10)",
.info_data_direction = SCST_DATA_NONE,
.info_op_flags = SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED|
SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED,
SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED|
SCST_SCSI_ATOMIC/* see comment in scst_cmd_overlap() */,
.get_cdb_info = get_cdb_info_none},
{.ops = 0x57, .devkey = "OOOOOOOOOOOOOOOO",
.info_op_name = "RELEASE(10)",
@@ -1292,8 +1300,9 @@ static const struct scst_sdbops scst_scsi_op_table[] = {
{.ops = 0x89, .devkey = "O ",
.info_op_name = "COMPARE AND WRITE",
.info_data_direction = SCST_DATA_WRITE,
.info_op_flags = SCST_TRANSFER_LEN_TYPE_FIXED|SCST_WRITE_MEDIUM|
SCST_SERIALIZED,
.info_op_flags = SCST_TRANSFER_LEN_TYPE_FIXED|
SCST_FULLY_LOCAL_CMD|SCST_LOCAL_CMD|
SCST_WRITE_MEDIUM|SCST_SCSI_ATOMIC,
.info_lba_off = 2, .info_lba_len = 8,
.info_len_off = 13, .info_len_len = 1,
.get_cdb_info = get_cdb_info_compare_and_write},
@@ -1999,10 +2008,13 @@ int scst_set_cmd_error_and_inf(struct scst_cmd *cmd, int key, int asc,
switch (cmd->sense[0] & 0x7f) {
case 0x70:
{
/* Fixed format */
uint32_t i = information;
cmd->sense[0] |= 0x80; /* Information field is valid */
put_unaligned_be32(information, &cmd->sense[3]);
put_unaligned_be32(i, &cmd->sense[3]);
break;
}
case 0x72:
/* Descriptor format */
cmd->sense[7] = 12; /* additional sense length */
@@ -3286,6 +3298,140 @@ static void scst_adjust_sg(struct scst_cmd *cmd, bool reg_sg,
return;
}
static bool __scst_adjust_sg_get_tail(struct scst_cmd *cmd,
struct scatterlist *sg, int *sg_cnt,
struct scatterlist **res_sg, int *res_sg_cnt,
int adjust_len, struct scst_orig_sg_data *orig_sg, int must_left)
{
int res = -ENOENT, i, j, l;
TRACE_ENTRY();
TRACE_DBG("cmd %p, sg_cnt %d, sg %p", cmd, *sg_cnt, sg);
l = 0;
for (i = 0, j = 0; i < *sg_cnt; i++, j++) {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
TRACE_DBG("i %d, j %d, sg %p, page_link %lx, len %d", i, j,
sg, sg[j].page_link, sg->length);
#else
TRACE_DBG("i %d, j %d, sg %p", i, j, sg);
#endif
if (unlikely(sg_is_chain(&sg[j]))) {
sg = sg_chain_ptr(&sg[j]);
j = 0;
}
l += sg[j].length;
if (l >= adjust_len) {
int offs = adjust_len - (l - sg[j].length);
TRACE_DBG_FLAG(TRACE_SG_OP|TRACE_MEMORY|TRACE_DEBUG,
"cmd %p (tag %llu), sg %p, adjust_len %d, i %d, "
"j %d, sg[j].length %d, offs %d",
cmd, (long long unsigned int)cmd->tag,
sg, adjust_len, i, j, sg[j].length, offs);
if (offs == sg[j].length) {
j++;
offs = 0;
}
orig_sg->p_orig_sg_cnt = sg_cnt;
orig_sg->orig_sg_cnt = *sg_cnt;
orig_sg->orig_sg_entry = &sg[j];
orig_sg->orig_entry_offs = sg[j].offset;
orig_sg->orig_entry_len = sg[j].length;
sg[j].offset += offs;
sg[j].length -= offs;
*res_sg = &sg[j];
*res_sg_cnt = *sg_cnt - j;
TRACE_DBG("j %d, sg %p, off %d, len %d, cnt %d "
"(offs %d)", j, &sg[j], sg[j].offset,
sg[j].length, *res_sg_cnt, offs);
res = 0;
break;
}
}
if (res != 0)
goto out;
#ifdef CONFIG_SCST_EXTRACHECKS
l = 0;
sg = *res_sg;
for (i = 0; i < *res_sg_cnt; i++)
l += sg[i].length;
if (l != must_left) {
PRINT_ERROR("Incorrect length %d of adjusted sg (cmd %p, "
"expected %d)", l, cmd, must_left);
res = -EINVAL;
scst_check_restore_sg_buff(cmd);
goto out;
}
#endif
out:
TRACE_EXIT_RES(res);
return res;
}
/*
* Returns in res_sg the tail of cmd's adjusted on adjust_len, i.e. tail
* of it. In res_sg_cnt sg_cnt of res_sg returned. Cmd only used to store
* cmd->sg restore information.
*
* Parameter must_left defines how many bytes must left in res_sg to consider
* operation successful.
*
* Returns 0 on success or error code otherwise.
*
* NOTE! Before scst_restore_sg_buff() called cmd->sg is corrupted and
* can NOT be used!
*/
static int scst_adjust_sg_get_tail(struct scst_cmd *cmd,
struct scatterlist **res_sg, int *res_sg_cnt,
struct scatterlist **res_dif_sg, int *res_dif_sg_cnt,
int adjust_len, int must_left)
{
int res;
TRACE_ENTRY();
EXTRACHECKS_BUG_ON(cmd->sg_buff_modified || cmd->dif_sg_buff_modified);
res = __scst_adjust_sg_get_tail(cmd, cmd->sg, &cmd->sg_cnt, res_sg,
res_sg_cnt, adjust_len, &cmd->orig_sg, must_left);
if (res != 0)
goto out;
cmd->sg_buff_modified = 1;
if (cmd->dif_sg != NULL) {
adjust_len >>= (cmd->dev->block_shift - SCST_DIF_TAG_SHIFT);
must_left >>= (cmd->dev->block_shift - SCST_DIF_TAG_SHIFT);
TRACE_DBG("DIF adjust_len %d, must_left %d", adjust_len, must_left);
res = __scst_adjust_sg_get_tail(cmd, cmd->dif_sg, &cmd->dif_sg_cnt,
res_dif_sg, res_dif_sg_cnt, adjust_len,
&cmd->orig_dif_sg, must_left);
if (res != 0) {
scst_restore_sg_buff(cmd);
goto out;
}
cmd->dif_sg_buff_modified = 1;
}
out:
TRACE_EXIT_RES(res);
return res;
}
/**
* scst_restore_sg_buff() - restores modified sg buffer
*
@@ -3977,6 +4123,7 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev)
#endif
scst_init_mem_lim(&dev->dev_mem_lim);
spin_lock_init(&dev->dev_lock);
INIT_LIST_HEAD(&dev->dev_exec_cmd_list);
INIT_LIST_HEAD(&dev->blocked_cmd_list);
INIT_LIST_HEAD(&dev->dev_tgt_dev_list);
INIT_LIST_HEAD(&dev->dev_acg_dev_list);
@@ -4007,6 +4154,9 @@ void scst_free_device(struct scst_device *dev)
{
TRACE_ENTRY();
EXTRACHECKS_BUG_ON(dev->dev_scsi_atomic_cmd_active != 0);
EXTRACHECKS_BUG_ON(!list_empty(&dev->dev_exec_cmd_list));
#ifdef CONFIG_SCST_EXTRACHECKS
if (!list_empty(&dev->dev_tgt_dev_list) ||
!list_empty(&dev->dev_acg_dev_list)) {
@@ -6133,6 +6283,258 @@ out_done:
}
EXPORT_SYMBOL_GPL(scst_write_same);
struct scst_cwr_priv {
/* Must be the first for scst_finish_internal_cmd()! */
scst_i_finish_fn_t cwr_finish_fn;
struct scst_cmd *cwr_orig_cmd;
};
static void scst_cwr_finished(struct scst_cwr_priv *cwrp)
{
struct scst_cmd *cwr_cmd = cwrp->cwr_orig_cmd;
TRACE_ENTRY();
TRACE_DBG("cwr cmd %p finished with status %d", cwr_cmd, cwr_cmd->status);
kfree(cwrp);
cwr_cmd->completed = 1; /* for success */
cwr_cmd->scst_cmd_done(cwr_cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD);
TRACE_EXIT();
return;
}
static void scst_cwr_write_cmd_finished(struct scst_cmd *cmd)
{
struct scst_cwr_priv *cwrp = cmd->tgt_i_priv;
struct scst_cmd *cwr_cmd = cwrp->cwr_orig_cmd;
TRACE_ENTRY();
TRACE_DBG("WRITE cmd %p finished (cwr cmd %p)", cmd, cwr_cmd);
EXTRACHECKS_BUG_ON(cmd->cdb[0] != WRITE_16);
if (cmd->status != 0) {
int rc;
TRACE_DBG("WRITE cmd %p (cwr cmd %p) finished not successfully",
cmd, cwr_cmd);
sBUG_ON(cmd->resp_data_len != 0);
if (cmd->status == SAM_STAT_CHECK_CONDITION)
rc = scst_set_cmd_error_sense(cwr_cmd, cmd->sense,
cmd->sense_valid_len);
else {
sBUG_ON(cmd->sense != NULL);
rc = scst_set_cmd_error_status(cwr_cmd, cmd->status);
}
if (rc != 0) {
/* Requeue possible UA */
if (scst_is_ua_sense(cmd->sense, cmd->sense_valid_len))
scst_requeue_ua(cmd, NULL, 0);
}
}
scst_cwr_finished(cwrp);
TRACE_EXIT();
return;
}
static void scst_cwr_read_cmd_finished(struct scst_cmd *cmd)
{
struct scst_cwr_priv *cwrp = cmd->tgt_i_priv;
struct scst_cmd *cwr_cmd = cwrp->cwr_orig_cmd;
bool c, dif;
uint8_t write16_cdb[16];
struct scst_cmd *wcmd;
int data_len, miscompare_offs, rc;
TRACE_ENTRY();
TRACE_DBG("READ cmd %p finished (cwr cmd %p)", cmd, cwr_cmd);
if (cmd->status != 0) {
int rc;
TRACE_DBG("Read cmd %p (cwr cmd %p) finished not successfully",
cmd, cwr_cmd);
sBUG_ON(cmd->resp_data_len != 0);
if (cmd->status == SAM_STAT_CHECK_CONDITION)
rc = scst_set_cmd_error_sense(cwr_cmd, cmd->sense,
cmd->sense_valid_len);
else {
sBUG_ON(cmd->sense != NULL);
rc = scst_set_cmd_error_status(cwr_cmd, cmd->status);
}
if (rc != 0) {
/* Requeue possible UA */
if (scst_is_ua_sense(cmd->sense, cmd->sense_valid_len))
scst_requeue_ua(cmd, NULL, 0);
}
goto out_finish;
}
if (unlikely(test_bit(SCST_CMD_ABORTED, &cwr_cmd->cmd_flags))) {
TRACE_MGMT_DBG("cwr cmd %p aborted", cwr_cmd);
goto out_finish;
}
c = sg_cmp(cmd->sg, cwr_cmd->sg, 0, 0, &miscompare_offs
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
, KM_USER0, KM_USER1
#endif
);
if (!c) {
scst_set_cmd_error_and_inf(cwr_cmd,
SCST_LOAD_SENSE(scst_sense_miscompare_error), miscompare_offs);
goto out_finish;
}
data_len = cwr_cmd->data_len;
/* We ignore the read PI checks as required by SBC */
rc = scst_dif_process_write(cmd);
if (unlikely(rc != 0))
goto out_finish;
memset(write16_cdb, 0, sizeof(write16_cdb));
write16_cdb[0] = WRITE_16;
write16_cdb[1] = cwr_cmd->cdb[1];
put_unaligned_be64(cwr_cmd->lba, &write16_cdb[2]);
put_unaligned_be32(data_len >> cmd->dev->block_shift, &write16_cdb[10]);
dif = scst_cmd_needs_dif_buf(cwr_cmd) && ((cwr_cmd->cdb[1] & 0xE0) != 0);
if (dif)
write16_cdb[1] |= cwr_cmd->cdb[1] & 0xE0;
wcmd = scst_create_prepare_internal_cmd(cwr_cmd, write16_cdb,
sizeof(write16_cdb), SCST_CMD_QUEUE_HEAD_OF_QUEUE);
if (wcmd == NULL)
goto out_busy;
wcmd->expected_data_direction = SCST_DATA_WRITE;
wcmd->expected_transfer_len_full = data_len;
if (dif)
wcmd->expected_transfer_len_full +=
data_len >> (cmd->dev->block_shift - SCST_DIF_TAG_SHIFT);
wcmd->expected_values_set = 1;
wcmd->tgt_i_priv = cwrp;
rc = scst_adjust_sg_get_tail(cwr_cmd, &wcmd->tgt_i_sg,
&wcmd->tgt_i_sg_cnt, &wcmd->tgt_i_dif_sg,
&wcmd->tgt_i_dif_sg_cnt, data_len, data_len);
/*
* It must not happen, because get_cdb_info_compare_and_write()
* supposed to ensure that.
*/
EXTRACHECKS_BUG_ON(rc != 0);
wcmd->tgt_i_data_buf_alloced = 1;
cwrp->cwr_finish_fn = scst_cwr_write_cmd_finished;
TRACE_DBG("Adding WRITE(16) wcmd %p to head of active cmd list", wcmd);
spin_lock_irq(&wcmd->cmd_threads->cmd_list_lock);
list_add(&wcmd->cmd_list_entry, &wcmd->cmd_threads->active_cmd_list);
wake_up(&wcmd->cmd_threads->cmd_list_waitQ);
spin_unlock_irq(&wcmd->cmd_threads->cmd_list_lock);
out:
TRACE_EXIT();
return;
out_busy:
scst_set_busy(cwr_cmd);
out_finish:
scst_cwr_finished(cwrp);
goto out;
}
int scst_cmp_wr_local(struct scst_cmd *cmd)
{
int res = SCST_EXEC_COMPLETED;
struct scst_cwr_priv *cwrp;
uint8_t read16_cdb[16];
struct scst_cmd *rcmd;
int data_len;
TRACE_ENTRY();
/* COMPARE AND WRITE is SBC only command */
EXTRACHECKS_BUG_ON(cmd->dev->type != TYPE_DISK);
cmd->status = 0;
cmd->msg_status = 0;
cmd->host_status = DID_OK;
cmd->driver_status = 0;
if (unlikely(cmd->bufflen == 0)) {
TRACE(TRACE_MINOR, "Zero bufflen (cmd %p)", cmd);
goto out_done;
}
/* ToDo: HWALIGN'ed kmem_cache */
cwrp = kzalloc(sizeof(*cwrp), GFP_KERNEL);
if (cwrp == NULL) {
PRINT_ERROR("Unable to allocate cwr_priv (size %zd, cmd %p)",
sizeof(*cwrp), cmd);
goto out_busy;
}
cwrp->cwr_orig_cmd = cmd;
cwrp->cwr_finish_fn = scst_cwr_read_cmd_finished;
data_len = cmd->data_len;
/*
* As required by SBC, DIF PI, if any, is not checked for the read part
*/
memset(read16_cdb, 0, sizeof(read16_cdb));
read16_cdb[0] = READ_16;
read16_cdb[1] = cmd->cdb[1] & ~0xE0; /* as required, see above */
put_unaligned_be64(cmd->lba, &read16_cdb[2]);
put_unaligned_be32(data_len >> cmd->dev->block_shift, &read16_cdb[10]);
rcmd = scst_create_prepare_internal_cmd(cmd, read16_cdb,
sizeof(read16_cdb), SCST_CMD_QUEUE_HEAD_OF_QUEUE);
if (rcmd == NULL) {
res = -ENOMEM;
goto out_free;
}
rcmd->expected_data_direction = SCST_DATA_READ;
rcmd->expected_transfer_len_full = data_len;
rcmd->expected_values_set = 1;
rcmd->tgt_i_priv = cwrp;
TRACE_DBG("Adding READ(16) cmd %p to head of active cmd list", rcmd);
spin_lock_irq(&rcmd->cmd_threads->cmd_list_lock);
list_add(&rcmd->cmd_list_entry, &rcmd->cmd_threads->active_cmd_list);
wake_up(&rcmd->cmd_threads->cmd_list_waitQ);
spin_unlock_irq(&rcmd->cmd_threads->cmd_list_lock);
out:
TRACE_EXIT_RES(res);
return res;
out_free:
kfree(cwrp);
out_busy:
scst_set_busy(cmd);
out_done:
cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD);
goto out;
}
int scst_finish_internal_cmd(struct scst_cmd *cmd)
{
int res;
@@ -6667,7 +7069,8 @@ void scst_free_cmd(struct scst_cmd *cmd)
if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
TRACE_MGMT_DBG("Freeing aborted cmd %p", cmd);
EXTRACHECKS_BUG_ON(cmd->unblock_dev || cmd->dec_on_dev_needed);
EXTRACHECKS_BUG_ON(cmd->unblock_dev || cmd->dec_on_dev_needed ||
cmd->on_dev_exec_list);
/*
* Target driver can already free sg buffer before calling
@@ -7703,6 +8106,243 @@ EXPORT_SYMBOL(scst_scsi_exec_async);
#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30) */
/*
* Can switch to the next dst_sg element, so, to cmp to strictly only
* one dst_sg element, it must be either last in the chain, or
* cmp_len == dst_sg->length.
*/
static int sg_cmp_elem(struct scatterlist **pdst_sg, size_t *pdst_len,
size_t *pdst_offs, struct scatterlist *src_sg,
size_t cmp_len, int *miscompare_offs,
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
enum km_type d_km_type, enum km_type s_km_type,
#endif
bool *cmp_res)
{
int res = 0;
struct scatterlist *dst_sg;
size_t src_len, dst_len, src_offs, dst_offs;
struct page *src_page, *dst_page;
void *saddr, *daddr;
*cmp_res = true;
dst_sg = *pdst_sg;
dst_len = *pdst_len;
dst_offs = *pdst_offs;
dst_page = sg_page(dst_sg);
src_page = sg_page(src_sg);
src_len = src_sg->length;
src_offs = src_sg->offset;
do {
size_t n;
int rc;
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
saddr = kmap_atomic(src_page + (src_offs >> PAGE_SHIFT), s_km_type) +
(src_offs & ~PAGE_MASK);
daddr = kmap_atomic(dst_page + (dst_offs >> PAGE_SHIFT), d_km_type) +
(dst_offs & ~PAGE_MASK);
#else
saddr = kmap_atomic(src_page + (src_offs >> PAGE_SHIFT)) +
(src_offs & ~PAGE_MASK);
daddr = kmap_atomic(dst_page + (dst_offs >> PAGE_SHIFT)) +
(dst_offs & ~PAGE_MASK);
#endif
if (((src_offs & ~PAGE_MASK) == 0) &&
((dst_offs & ~PAGE_MASK) == 0) &&
(src_len >= PAGE_SIZE) && (dst_len >= PAGE_SIZE) &&
(cmp_len >= PAGE_SIZE))
n = PAGE_SIZE;
else {
n = min_t(size_t, PAGE_SIZE - (dst_offs & ~PAGE_MASK),
PAGE_SIZE - (src_offs & ~PAGE_MASK));
n = min(n, src_len);
n = min(n, dst_len);
n = min_t(size_t, n, cmp_len);
}
/* Optimized for rare miscompares */
rc = memcmp(daddr, saddr, n);
if (rc != 0) {
*cmp_res = false;
if (miscompare_offs != NULL) {
int i, len = n;
unsigned long saddr_int = (unsigned long)saddr;
unsigned long daddr_int = (unsigned long)daddr;
int align_size = sizeof(unsigned long long);
int align_mask = align_size-1;
*miscompare_offs = -1;
if (((saddr_int & align_mask) == 0) &&
((daddr_int & align_mask) == 0) &&
((len & align_mask) == 0)) {
/* Fast path: both buffers and len aligned */
unsigned long long *s = saddr;
unsigned long long *d = daddr;
EXTRACHECKS_BUG_ON(len % align_size != 0);
len /= align_size;
for (i = 0; i < len; i++) {
if (s[i] != d[i]) {
uint8_t *s8 = (uint8_t *)&s[i];
uint8_t *d8 = (uint8_t *)&d[i];
int j;
for (j = 0; j < align_size; j++) {
if (s8[j] != d8[j]) {
*miscompare_offs = i * align_size + j;
break;
}
}
break;
}
}
} else {
uint8_t *s = saddr;
uint8_t *d = daddr;
for (i = 0; i < len; i++) {
if (s[i] != d[i]) {
*miscompare_offs = i;
break;
}
}
}
EXTRACHECKS_BUG_ON(*miscompare_offs == -1);
}
goto out_unmap;
}
dst_offs += n;
src_offs += n;
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
kunmap_atomic(saddr, s_km_type);
kunmap_atomic(daddr, d_km_type);
#else
kunmap_atomic(saddr);
kunmap_atomic(daddr);
#endif
res += n;
cmp_len -= n;
if (cmp_len == 0)
goto out;
src_len -= n;
dst_len -= n;
if (dst_len == 0) {
dst_sg = sg_next_inline(dst_sg);
if (dst_sg == NULL)
goto out;
dst_page = sg_page(dst_sg);
dst_len = dst_sg->length;
dst_offs = dst_sg->offset;
}
} while (src_len > 0);
out:
*pdst_sg = dst_sg;
*pdst_len = dst_len;
*pdst_offs = dst_offs;
return res;
out_unmap:
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
kunmap_atomic(saddr, s_km_type);
kunmap_atomic(daddr, d_km_type);
#else
kunmap_atomic(saddr);
kunmap_atomic(daddr);
#endif
goto out;
}
/**
* sg_cmp - compare 2 SG vectors
* @dst_sg: SG 1
* @src_sg: SG 2
* @nents_to_cmp: maximum number of entries to compare
* @cmp_len: maximum amount of data to compare. If 0, then compare all.
* @miscompare_offs: offset of the first miscompare. Can be NULL.
* @d_km_type: kmap_atomic type for SG 1
* @s_km_type: kmap_atomic type for SG 2
*
* Description:
* Data from the first SG vector will be compired with the second SG
* vector. End of the vectors will be determined by sg_next() returning
* NULL. Returns true if both vectors have identical data, false otherwise.
*
* Note! It ignores size of the vectors, so SGs with different size with
* the same data in min(sg1_size, sg2_size) size will match!
*/
static bool sg_cmp(struct scatterlist *dst_sg, struct scatterlist *src_sg,
int nents_to_cmp, size_t cmp_len, int *miscompare_offs
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
, enum km_type d_km_type, enum km_type s_km_type
#endif
)
{
bool res = true;
size_t dst_len, dst_offs;
int compared_all = 0;
TRACE_ENTRY();
TRACE_DBG("dst_sg %p, src_sg %p, nents_to_cmp %d, cmp_len %zd",
dst_sg, src_sg, nents_to_cmp, cmp_len);
if (cmp_len == 0)
cmp_len = 0x7FFFFFFF; /* cmp all */
if (nents_to_cmp == 0)
nents_to_cmp = 0x7FFFFFFF; /* cmp all */
dst_len = dst_sg->length;
dst_offs = dst_sg->offset;
do {
int compared;
TRACE_DBG("dst_sg %p, dst_len %zd, dst_offs %zd, src_sg %p, "
"nents_to_cmp %d, cmp_len %zd, compared_all %d",
dst_sg, dst_len, dst_offs, src_sg, nents_to_cmp,
cmp_len, compared_all);
compared = sg_cmp_elem(&dst_sg, &dst_len, &dst_offs,
src_sg, cmp_len, miscompare_offs,
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
d_km_type, s_km_type,
#endif
&res);
if (!res) {
if (miscompare_offs != NULL) {
*miscompare_offs += compared_all;
TRACE_DBG("miscompare_offs %d",
*miscompare_offs);
}
goto out;
}
cmp_len -= compared;
compared_all += compared;
if ((cmp_len == 0) || (dst_sg == NULL))
goto out;
nents_to_cmp--;
if (nents_to_cmp == 0)
goto out;
src_sg = sg_next_inline(src_sg);
} while (src_sg != NULL);
out:
TRACE_EXIT_RES(res);
return res;
}
/**
* scst_copy_sg() - copy data between the command's SGs
*
scst/src/scst_priv.h
+23
View File
@@ -808,6 +808,8 @@ void scst_unthrottle_cmd(struct scst_cmd *cmd);
int scst_do_internal_parsing(struct scst_cmd *cmd);
int scst_parse_descriptors(struct scst_cmd *cmd);
int scst_cmp_wr_local(struct scst_cmd *cmd);
int scst_pr_init(struct scst_device *dev);
void scst_pr_cleanup(struct scst_device *dev);
@@ -819,6 +821,27 @@ void scst_vfs_unlink_and_put(struct path *path);
int scst_copy_file(const char *src, const char *dest);
struct scst_cmd *__scst_create_prepare_internal_cmd(const uint8_t *cdb,
unsigned int cdb_len, enum scst_cmd_queue_type queue_type,
struct scst_tgt_dev *tgt_dev, gfp_t gfp_mask, bool fantom);
static inline bool scst_lba1_inside_lba2(int64_t lba1,
int64_t lba2, int64_t lba2_blocks)
{
bool res;
TRACE_DBG("lba1 %lld, lba2 %lld, lba2_blocks %lld", (long long)lba1,
(long long)lba2, (long long)lba2_blocks);
if ((lba1 >= lba2) && (lba1 < (lba2 + lba2_blocks)))
res = true;
else
res = false;
TRACE_EXIT_RES(res);
return res;
}
#ifdef CONFIG_SCST_DEBUG_TM
extern void tm_dbg_check_released_cmds(void);
extern int tm_dbg_check_cmd(struct scst_cmd *cmd);
scst/src/scst_targ.c
+388 -14
View File
@@ -103,25 +103,296 @@ static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
return;
}
/* No locks */
static bool scst_check_blocked_dev(struct scst_cmd *cmd)
static bool scst_unmap_overlap(struct scst_cmd *cmd, int64_t lba2,
int64_t lba2_blocks)
{
bool res = false;
struct scst_data_descriptor *pd = cmd->cmd_data_descriptors;
int i;
TRACE_ENTRY();
if (pd == NULL)
goto out;
for (i = 0; pd[i].sdd_blocks != 0; i++) {
struct scst_data_descriptor *d = &pd[i];
TRACE_DBG("i %d, lba %lld, blocks %lld", i,
(long long)d->sdd_lba, (long long)d->sdd_blocks);
res = scst_lba1_inside_lba2(d->sdd_lba, lba2, lba2_blocks);
if (res)
goto out;
res = scst_lba1_inside_lba2(lba2, d->sdd_lba, d->sdd_blocks);
if (res)
goto out;
}
out:
TRACE_EXIT_RES(res);
return res;
}
static bool scst_cmd_overlap_cwr(struct scst_cmd *cwr_cmd, struct scst_cmd *cmd)
{
bool res;
TRACE_ENTRY();
TRACE_DBG("cwr_cmd %p, cmd %p (op %s, LBA valid %d, lba %lld, "
"len %lld)", cwr_cmd, cmd, scst_get_opcode_name(cmd),
(cmd->op_flags & SCST_LBA_NOT_VALID) == 0,
(long long)cmd->lba, (long long)cmd->data_len);
EXTRACHECKS_BUG_ON(cwr_cmd->cdb[0] != COMPARE_AND_WRITE);
/*
* In addition to requirements listed in "Model for uninterrupted
* sequences on LBA ranges" (SBC) VMware wants that COMPARE AND WRITE
* be atomic against RESERVEs, as well as RESERVEs be atomic against
* all COMPARE AND WRITE commands and only against them.
*/
if (cmd->op_flags & SCST_LBA_NOT_VALID) {
switch (cmd->cdb[0]) {
case RESERVE:
case RESERVE_10:
res = true;
break;
case UNMAP:
res = scst_unmap_overlap(cmd, cwr_cmd->lba,
cwr_cmd->data_len);
break;
default:
res = false;
break;
}
goto out;
}
/* If LBA valid, block_shift must be valid */
EXTRACHECKS_BUG_ON(cmd->dev->block_shift <= 0);
res = scst_lba1_inside_lba2(cwr_cmd->lba, cmd->lba,
cmd->data_len >> cmd->dev->block_shift);
if (res)
goto out;
res = scst_lba1_inside_lba2(cmd->lba, cwr_cmd->lba,
cwr_cmd->data_len >> cwr_cmd->dev->block_shift);
out:
TRACE_EXIT_RES(res);
return res;
}
static bool scst_cmd_overlap_reserve(struct scst_cmd *reserve_cmd,
struct scst_cmd *cmd)
{
bool res;
TRACE_ENTRY();
TRACE_DBG("reserve_cmd %p, cmd %p (op %s, LBA valid %d, lba %lld, "
"len %lld)", reserve_cmd, cmd, scst_get_opcode_name(cmd),
(cmd->op_flags & SCST_LBA_NOT_VALID) == 0,
(long long)cmd->lba, (long long)cmd->data_len);
EXTRACHECKS_BUG_ON((reserve_cmd->cdb[0] != RESERVE) &&
(reserve_cmd->cdb[0] != RESERVE_10));
/*
* In addition to requirements listed in "Model for uninterrupted
* sequences on LBA ranges" (SBC) VMware wants that COMPARE AND WRITE
* be atomic against RESERVEs, as well as RESERVEs be atomic against
* all COMPARE AND WRITE commands and only against them.
*/
if (cmd->cdb[0] == COMPARE_AND_WRITE)
res = true;
else
res = false;
TRACE_EXIT_RES(res);
return res;
}
static bool scst_cmd_overlap_atomic(struct scst_cmd *atomic_cmd, struct scst_cmd *cmd)
{
bool res;
TRACE_ENTRY();
TRACE_DBG("atomic_cmd %p (op %s), cmd %p (op %s)", atomic_cmd,
scst_get_opcode_name(atomic_cmd), cmd, scst_get_opcode_name(cmd));
EXTRACHECKS_BUG_ON((atomic_cmd->op_flags & SCST_SCSI_ATOMIC) == 0);
switch (atomic_cmd->cdb[0]) {
case COMPARE_AND_WRITE:
res = scst_cmd_overlap_cwr(atomic_cmd, cmd);
break;
case RESERVE:
case RESERVE_10:
res = scst_cmd_overlap_reserve(atomic_cmd, cmd);
break;
default:
res = false;
break;
}
TRACE_EXIT_RES(res);
return res;
}
static bool scst_cmd_overlap(struct scst_cmd *chk_cmd, struct scst_cmd *cmd)
{
bool res = false;
TRACE_ENTRY();
TRACE_DBG("chk_cmd %p, cmd %p", chk_cmd, cmd);
if ((chk_cmd->op_flags & SCST_SCSI_ATOMIC) != 0)
res = scst_cmd_overlap_atomic(chk_cmd, cmd);
else if ((cmd->op_flags & SCST_SCSI_ATOMIC) != 0)
res = scst_cmd_overlap_atomic(cmd, chk_cmd);
else
res = false;
TRACE_EXIT_RES(res);
return res;
}
/*
* dev_lock supposed to be held and BH disabled. Returns true if cmd blocked,
* hence stop processing it and go to the next command.
*/
static bool scst_check_scsi_atomicity(struct scst_cmd *chk_cmd)
{
bool res = false;
struct scst_device *dev = chk_cmd->dev;
struct scst_cmd *cmd;
TRACE_ENTRY();
/*
* chk_cmd isn't necessary SCSI atomic! For instance, if another SCSI
* atomic cmd is waiting blocked.
*/
TRACE_DBG("chk_cmd %p (op %s, internal %d, lba %lld, len %lld)",
chk_cmd, scst_get_opcode_name(chk_cmd), chk_cmd->internal,
(long long)chk_cmd->lba, (long long)chk_cmd->data_len);
list_for_each_entry(cmd, &dev->dev_exec_cmd_list, dev_exec_cmd_list_entry) {
if (chk_cmd == cmd)
continue;
if (scst_cmd_overlap(chk_cmd, cmd)) {
struct scst_cmd **p = cmd->scsi_atomic_blocked_cmds;
/*
* kmalloc() allocates by at least 32 bytes increments,
* hence krealloc() on 8 bytes increments, if not all
* that space is used, does nothing.
*/
p = krealloc(p, sizeof(*p) * (cmd->scsi_atomic_blocked_cmds_count + 1),
GFP_ATOMIC);
if (p == NULL)
goto out_busy_undo;
p[cmd->scsi_atomic_blocked_cmds_count] = chk_cmd;
cmd->scsi_atomic_blocked_cmds = p;
cmd->scsi_atomic_blocked_cmds_count++;
chk_cmd->scsi_atomic_blockers++;
TRACE_BLOCK("Delaying cmd %p (op %s, lba %lld, "
"len %lld, blockers %d) due to overlap with "
"cmd %p (op %s, lba %lld, len %lld, blocked "
"cmds %d)", chk_cmd, scst_get_opcode_name(chk_cmd),
(long long)chk_cmd->lba,
(long long)chk_cmd->data_len,
chk_cmd->scsi_atomic_blockers, cmd,
scst_get_opcode_name(cmd), (long long)cmd->lba,
(long long)cmd->data_len,
cmd->scsi_atomic_blocked_cmds_count);
res = true;
}
}
out:
TRACE_EXIT_RES(res);
return res;
out_busy_undo:
list_for_each_entry(cmd, &dev->dev_exec_cmd_list, dev_exec_cmd_list_entry) {
struct scst_cmd **p = cmd->scsi_atomic_blocked_cmds;
if ((p != NULL) && (p[cmd->scsi_atomic_blocked_cmds_count-1] == chk_cmd)) {
cmd->scsi_atomic_blocked_cmds_count--;
chk_cmd->scsi_atomic_blockers--;
}
}
sBUG_ON(chk_cmd->scsi_atomic_blockers != 0);
scst_set_busy(chk_cmd);
scst_set_cmd_abnormal_done_state(chk_cmd);
spin_lock_irq(&chk_cmd->cmd_threads->cmd_list_lock);
TRACE_MGMT_DBG("Adding on error chk_cmd %p back to head of active cmd "
"list", chk_cmd);
list_add(&chk_cmd->cmd_list_entry, &chk_cmd->cmd_threads->active_cmd_list);
wake_up(&chk_cmd->cmd_threads->cmd_list_waitQ);
spin_unlock_irq(&chk_cmd->cmd_threads->cmd_list_lock);
res = false;
goto out;
}
/*
* dev_lock supposed to be BH locked. Returns true if cmd blocked, hence stop
* processing it and go to the next command.
*/
bool scst_do_check_blocked_dev(struct scst_cmd *cmd)
{
bool res;
struct scst_device *dev = cmd->dev;
TRACE_ENTRY();
if (unlikely(cmd->internal)) {
/*
* The original command can already block the device and must
* hold reference to it, so internal command should always pass.
*/
sBUG_ON(dev->on_dev_cmd_count == 0);
res = false;
goto out;
/*
* We want to have fairness between just unblocked previously blocked
* SCSI atomic cmds and new cmds came after them. Otherwise, the new
* cmds can bypass the SCSI atomic cmds and make them unfairly wait
* again. So, we need to always, from the beginning, have blocked SCSI
* atomic cmds on the exec list, even if they blocked, as well
* as dev's SCSI atomic cmds counter incremented.
*/
if (likely(!cmd->on_dev_exec_list)) {
list_add_tail(&cmd->dev_exec_cmd_list_entry, &dev->dev_exec_cmd_list);
cmd->on_dev_exec_list = 1;
}
spin_lock_bh(&dev->dev_lock);
/*
* After a cmd passed SCSI atomicy check, there's no need to recheck SCSI
* atomicity for this cmd in future entrances here, because then all
* future overlapping with this cmd cmds will be blocked on it.
*/
if (unlikely(((cmd->op_flags & SCST_SCSI_ATOMIC) != 0) ||
(dev->dev_scsi_atomic_cmd_active != 0)) &&
!cmd->scsi_atomicity_checked) {
cmd->scsi_atomicity_checked = 1;
if ((cmd->op_flags & SCST_SCSI_ATOMIC) != 0) {
dev->dev_scsi_atomic_cmd_active++;
TRACE_DBG("cmd %p (dev %p), scsi atomic_cmd_active %d",
cmd, dev, dev->dev_scsi_atomic_cmd_active);
}
res = scst_check_scsi_atomicity(cmd);
if (res) {
EXTRACHECKS_BUG_ON(dev->dev_scsi_atomic_cmd_active == 0);
goto out;
}
}
dev->on_dev_cmd_count++;
cmd->dec_on_dev_needed = 1;
@@ -140,8 +411,37 @@ static bool scst_check_blocked_dev(struct scst_cmd *cmd)
cmd->dec_on_dev_needed = 0;
TRACE_DBG("New dec on_dev_count %d (cmd %p)",
dev->on_dev_cmd_count, cmd);
goto out;
}
out:
TRACE_EXIT_RES(res);
return res;
}
/*
* No locks. Returns true if cmd blocked, hence stop processing it and go to
* the next command.
*/
static bool scst_check_blocked_dev(struct scst_cmd *cmd)
{
bool res;
struct scst_device *dev = cmd->dev;
TRACE_ENTRY();
if (unlikely(cmd->internal)) {
/*
* The original command can already block the device and must
* hold reference to it, so internal command should always pass.
*/
sBUG_ON(dev->on_dev_cmd_count == 0);
res = false;
goto out;
}
spin_lock_bh(&dev->dev_lock);
res = scst_do_check_blocked_dev(cmd);
spin_unlock_bh(&dev->dev_lock);
out:
@@ -149,12 +449,67 @@ out:
return res;
}
/* No locks */
static void scst_check_unblock_dev(struct scst_cmd *cmd)
/* dev_lock supposed to be held and BH disabled */
static void scst_check_unblock_scsi_atomic_cmds(struct scst_cmd *cmd)
{
int i;
TRACE_ENTRY();
EXTRACHECKS_BUG_ON(cmd->scsi_atomic_blocked_cmds_count == 0);
for (i = 0; i < cmd->scsi_atomic_blocked_cmds_count; i++) {
struct scst_cmd *acmd = cmd->scsi_atomic_blocked_cmds[i];
acmd->scsi_atomic_blockers--;
if (acmd->scsi_atomic_blockers == 0) {
TRACE_BLOCK("Unblocking blocked acmd %p (blocker "
"cmd %p)", acmd, cmd);
spin_lock_irq(&acmd->cmd_threads->cmd_list_lock);
if (acmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE)
list_add(&acmd->cmd_list_entry,
&acmd->cmd_threads->active_cmd_list);
else
list_add_tail(&acmd->cmd_list_entry,
&acmd->cmd_threads->active_cmd_list);
wake_up(&acmd->cmd_threads->cmd_list_waitQ);
spin_unlock_irq(&acmd->cmd_threads->cmd_list_lock);
}
}
kfree(cmd->scsi_atomic_blocked_cmds);
cmd->scsi_atomic_blocked_cmds = NULL;
cmd->scsi_atomic_blocked_cmds_count = 0;
TRACE_EXIT();
return;
}
/* dev_lock supposed to be BH locked */
void __scst_check_unblock_dev(struct scst_cmd *cmd)
{
struct scst_device *dev = cmd->dev;
spin_lock_bh(&dev->dev_lock);
TRACE_ENTRY();
/*
* We might be called here as part of Copy Manager's check blocking
* undo, so restore all flags in the previous state to allow
* restart of this cmd.
*/
if (likely(cmd->on_dev_exec_list)) {
list_del(&cmd->dev_exec_cmd_list_entry);
cmd->on_dev_exec_list = 0;
}
if (unlikely((cmd->op_flags & SCST_SCSI_ATOMIC) != 0)) {
if (likely(cmd->scsi_atomicity_checked)) {
dev->dev_scsi_atomic_cmd_active--;
TRACE_DBG("cmd %p, scsi atomic_cmd_active %d",
cmd, dev->dev_scsi_atomic_cmd_active);
cmd->scsi_atomicity_checked = 0;
}
}
if (likely(cmd->dec_on_dev_needed)) {
dev->on_dev_cmd_count--;
@@ -163,6 +518,9 @@ static void scst_check_unblock_dev(struct scst_cmd *cmd)
dev->on_dev_cmd_count, cmd);
}
if (unlikely(cmd->scsi_atomic_blocked_cmds != NULL))
scst_check_unblock_scsi_atomic_cmds(cmd);
if (unlikely(cmd->unblock_dev)) {
TRACE_BLOCK("cmd %p (tag %llu): unblocking dev %s", cmd,
(unsigned long long int)cmd->tag, dev->virt_name);
@@ -184,7 +542,22 @@ static void scst_check_unblock_dev(struct scst_cmd *cmd)
}
}
TRACE_EXIT();
return;
}
/* No locks */
void scst_check_unblock_dev(struct scst_cmd *cmd)
{
struct scst_device *dev = cmd->dev;
TRACE_ENTRY();
spin_lock_bh(&dev->dev_lock);
__scst_check_unblock_dev(cmd);
spin_unlock_bh(&dev->dev_lock);
TRACE_EXIT();
return;
}
@@ -3145,6 +3518,7 @@ static scst_local_exec_fn scst_local_fns[256] = {
[PERSISTENT_RESERVE_OUT] = scst_persistent_reserve_out_local,
[REPORT_LUNS] = scst_report_luns_local,
[REQUEST_SENSE] = scst_request_sense_local,
[COMPARE_AND_WRITE] = scst_cmp_wr_local,
[MAINTENANCE_IN] = scst_maintenance_in,
};