diff --git a/scst/include/scst.h b/scst/include/scst.h index d968aa16f..80645c437 100644 --- a/scst/include/scst.h +++ b/scst/include/scst.h @@ -2136,6 +2136,12 @@ struct scst_cmd { /* Set if scst_dec_on_dev_cmd() call is needed on the cmd's finish */ unsigned int dec_on_dev_needed:1; + /* Set if cmd is on dev's exec_cmd_list */ + unsigned int on_dev_exec_list:1; + + /* Set if this cmd passed check for SCSI atomicity */ + unsigned int scsi_atomicity_checked:1; + /* Set if cmd is queued as hw pending */ unsigned int cmd_hw_pending:1; @@ -2298,6 +2304,15 @@ struct scst_cmd { unsigned short cdb_len; uint8_t cdb_buf[SCST_MAX_CDB_SIZE]; + /* List entry for dev's dev_exec_cmd_list */ + struct list_head dev_exec_cmd_list_entry; + + /* + * Array of blocked by this cmd SCSI atomic cmds with size + * scsi_atomic_blocked_cmds_count. Protected by dev->dev_lock. + */ + struct scst_cmd **scsi_atomic_blocked_cmds; + uint8_t lba_off; /* LBA offset in cdb */ uint8_t lba_len; /* LBA length in cdb */ uint8_t len_off; /* length offset in cdb */ @@ -2429,6 +2444,18 @@ struct scst_cmd { /* Used for storage of dev handler private stuff */ void *dh_priv; + /* + * Number of waiting for this cmd to finish commands + * with SCSI atomic guarantees. Protected by dev->dev_lock. + */ + int scsi_atomic_blockers; + + /* + * How many SCSI atomic cmds this cmd blocked, i.e. size of array + * scsi_atomic_blocked_cmds. Protected by dev->dev_lock. + */ + int scsi_atomic_blocked_cmds_count; + /* List entry for dev's blocked_cmd_list */ struct list_head blocked_cmd_list_entry; @@ -2812,6 +2839,17 @@ struct scst_device { */ int on_dev_cmd_count; + /* + * How many atomic SCSI commands being executed. Protected by dev_lock. + */ + int dev_scsi_atomic_cmd_active; + + /* + * List of all being executed on the dev commands. + * Protected by dev_lock. + */ + struct list_head dev_exec_cmd_list; + /* Memory limits for this device */ struct scst_mem_lim dev_mem_lim; diff --git a/scst/include/scst_const.h b/scst/include/scst_const.h index 1552b8e87..88327e234 100644 --- a/scst/include/scst_const.h +++ b/scst/include/scst_const.h @@ -228,9 +228,10 @@ enum scst_cdb_flags { #ifdef CONFIG_SCST_TEST_IO_IN_SIRQ SCST_TEST_IO_IN_SIRQ_ALLOWED = 0x8000, #endif - SCST_SERIALIZED = 0x10000, - SCST_STRICTLY_SERIALIZED = 0x20000|SCST_SERIALIZED, - SCST_DESCRIPTORS_BASED = 0x40000, + SCST_SERIALIZED = 0x10000, + SCST_STRICTLY_SERIALIZED = 0x20000|SCST_SERIALIZED, + SCST_DESCRIPTORS_BASED = 0x40000, + SCST_SCSI_ATOMIC = 0x100000, }; /************************************************************* diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index 91130a5e8..f883c4633 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -155,6 +155,12 @@ static int sg_copy(struct scatterlist *dst_sg, struct scatterlist *src_sg, #endif static void scst_free_descriptors(struct scst_cmd *cmd); +static bool sg_cmp(struct scatterlist *dst_sg, struct scatterlist *src_sg, + int nents_to_cmp, size_t cmp_len, int *miscompare_offs +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) + , enum km_type d_km_type, enum km_type s_km_type +#endif + ); const struct scst_opcode_descriptor scst_op_descr_inquiry = { .od_opcode = INQUIRY, @@ -757,7 +763,8 @@ static const struct scst_sdbops scst_scsi_op_table[] = { .info_op_name = "RESERVE", .info_data_direction = SCST_DATA_NONE, .info_op_flags = SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED| - SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED, + SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED| + SCST_SCSI_ATOMIC/* see comment in scst_cmd_overlap() */, .get_cdb_info = get_cdb_info_none}, {.ops = 0x17, .devkey = "MMMMMMMMMMMMMMMM", .info_op_name = "RELEASE", @@ -1170,7 +1177,8 @@ static const struct scst_sdbops scst_scsi_op_table[] = { .info_op_name = "RESERVE(10)", .info_data_direction = SCST_DATA_NONE, .info_op_flags = SCST_SMALL_TIMEOUT|SCST_LOCAL_CMD|SCST_SERIALIZED| - SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED, + SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED| + SCST_SCSI_ATOMIC/* see comment in scst_cmd_overlap() */, .get_cdb_info = get_cdb_info_none}, {.ops = 0x57, .devkey = "OOOOOOOOOOOOOOOO", .info_op_name = "RELEASE(10)", @@ -1292,8 +1300,9 @@ static const struct scst_sdbops scst_scsi_op_table[] = { {.ops = 0x89, .devkey = "O ", .info_op_name = "COMPARE AND WRITE", .info_data_direction = SCST_DATA_WRITE, - .info_op_flags = SCST_TRANSFER_LEN_TYPE_FIXED|SCST_WRITE_MEDIUM| - SCST_SERIALIZED, + .info_op_flags = SCST_TRANSFER_LEN_TYPE_FIXED| + SCST_FULLY_LOCAL_CMD|SCST_LOCAL_CMD| + SCST_WRITE_MEDIUM|SCST_SCSI_ATOMIC, .info_lba_off = 2, .info_lba_len = 8, .info_len_off = 13, .info_len_len = 1, .get_cdb_info = get_cdb_info_compare_and_write}, @@ -1999,10 +2008,13 @@ int scst_set_cmd_error_and_inf(struct scst_cmd *cmd, int key, int asc, switch (cmd->sense[0] & 0x7f) { case 0x70: + { /* Fixed format */ + uint32_t i = information; cmd->sense[0] |= 0x80; /* Information field is valid */ - put_unaligned_be32(information, &cmd->sense[3]); + put_unaligned_be32(i, &cmd->sense[3]); break; + } case 0x72: /* Descriptor format */ cmd->sense[7] = 12; /* additional sense length */ @@ -3286,6 +3298,140 @@ static void scst_adjust_sg(struct scst_cmd *cmd, bool reg_sg, return; } +static bool __scst_adjust_sg_get_tail(struct scst_cmd *cmd, + struct scatterlist *sg, int *sg_cnt, + struct scatterlist **res_sg, int *res_sg_cnt, + int adjust_len, struct scst_orig_sg_data *orig_sg, int must_left) +{ + int res = -ENOENT, i, j, l; + + TRACE_ENTRY(); + + TRACE_DBG("cmd %p, sg_cnt %d, sg %p", cmd, *sg_cnt, sg); + + l = 0; + for (i = 0, j = 0; i < *sg_cnt; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24) + TRACE_DBG("i %d, j %d, sg %p, page_link %lx, len %d", i, j, + sg, sg[j].page_link, sg->length); +#else + TRACE_DBG("i %d, j %d, sg %p", i, j, sg); +#endif + if (unlikely(sg_is_chain(&sg[j]))) { + sg = sg_chain_ptr(&sg[j]); + j = 0; + } + l += sg[j].length; + if (l >= adjust_len) { + int offs = adjust_len - (l - sg[j].length); + + TRACE_DBG_FLAG(TRACE_SG_OP|TRACE_MEMORY|TRACE_DEBUG, + "cmd %p (tag %llu), sg %p, adjust_len %d, i %d, " + "j %d, sg[j].length %d, offs %d", + cmd, (long long unsigned int)cmd->tag, + sg, adjust_len, i, j, sg[j].length, offs); + + if (offs == sg[j].length) { + j++; + offs = 0; + } + + orig_sg->p_orig_sg_cnt = sg_cnt; + orig_sg->orig_sg_cnt = *sg_cnt; + orig_sg->orig_sg_entry = &sg[j]; + orig_sg->orig_entry_offs = sg[j].offset; + orig_sg->orig_entry_len = sg[j].length; + + sg[j].offset += offs; + sg[j].length -= offs; + *res_sg = &sg[j]; + *res_sg_cnt = *sg_cnt - j; + + TRACE_DBG("j %d, sg %p, off %d, len %d, cnt %d " + "(offs %d)", j, &sg[j], sg[j].offset, + sg[j].length, *res_sg_cnt, offs); + + res = 0; + break; + } + } + + if (res != 0) + goto out; + +#ifdef CONFIG_SCST_EXTRACHECKS + l = 0; + sg = *res_sg; + for (i = 0; i < *res_sg_cnt; i++) + l += sg[i].length; + + if (l != must_left) { + PRINT_ERROR("Incorrect length %d of adjusted sg (cmd %p, " + "expected %d)", l, cmd, must_left); + res = -EINVAL; + scst_check_restore_sg_buff(cmd); + goto out; + } +#endif + +out: + TRACE_EXIT_RES(res); + return res; +} + +/* + * Returns in res_sg the tail of cmd's adjusted on adjust_len, i.e. tail + * of it. In res_sg_cnt sg_cnt of res_sg returned. Cmd only used to store + * cmd->sg restore information. + * + * Parameter must_left defines how many bytes must left in res_sg to consider + * operation successful. + * + * Returns 0 on success or error code otherwise. + * + * NOTE! Before scst_restore_sg_buff() called cmd->sg is corrupted and + * can NOT be used! + */ +static int scst_adjust_sg_get_tail(struct scst_cmd *cmd, + struct scatterlist **res_sg, int *res_sg_cnt, + struct scatterlist **res_dif_sg, int *res_dif_sg_cnt, + int adjust_len, int must_left) +{ + int res; + + TRACE_ENTRY(); + + EXTRACHECKS_BUG_ON(cmd->sg_buff_modified || cmd->dif_sg_buff_modified); + + res = __scst_adjust_sg_get_tail(cmd, cmd->sg, &cmd->sg_cnt, res_sg, + res_sg_cnt, adjust_len, &cmd->orig_sg, must_left); + if (res != 0) + goto out; + + cmd->sg_buff_modified = 1; + + if (cmd->dif_sg != NULL) { + adjust_len >>= (cmd->dev->block_shift - SCST_DIF_TAG_SHIFT); + must_left >>= (cmd->dev->block_shift - SCST_DIF_TAG_SHIFT); + + TRACE_DBG("DIF adjust_len %d, must_left %d", adjust_len, must_left); + + res = __scst_adjust_sg_get_tail(cmd, cmd->dif_sg, &cmd->dif_sg_cnt, + res_dif_sg, res_dif_sg_cnt, adjust_len, + &cmd->orig_dif_sg, must_left); + if (res != 0) { + scst_restore_sg_buff(cmd); + goto out; + } + + cmd->dif_sg_buff_modified = 1; + } + +out: + TRACE_EXIT_RES(res); + return res; +} + /** * scst_restore_sg_buff() - restores modified sg buffer * @@ -3977,6 +4123,7 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev) #endif scst_init_mem_lim(&dev->dev_mem_lim); spin_lock_init(&dev->dev_lock); + INIT_LIST_HEAD(&dev->dev_exec_cmd_list); INIT_LIST_HEAD(&dev->blocked_cmd_list); INIT_LIST_HEAD(&dev->dev_tgt_dev_list); INIT_LIST_HEAD(&dev->dev_acg_dev_list); @@ -4007,6 +4154,9 @@ void scst_free_device(struct scst_device *dev) { TRACE_ENTRY(); + EXTRACHECKS_BUG_ON(dev->dev_scsi_atomic_cmd_active != 0); + EXTRACHECKS_BUG_ON(!list_empty(&dev->dev_exec_cmd_list)); + #ifdef CONFIG_SCST_EXTRACHECKS if (!list_empty(&dev->dev_tgt_dev_list) || !list_empty(&dev->dev_acg_dev_list)) { @@ -6133,6 +6283,258 @@ out_done: } EXPORT_SYMBOL_GPL(scst_write_same); +struct scst_cwr_priv { + /* Must be the first for scst_finish_internal_cmd()! */ + scst_i_finish_fn_t cwr_finish_fn; + + struct scst_cmd *cwr_orig_cmd; +}; + +static void scst_cwr_finished(struct scst_cwr_priv *cwrp) +{ + struct scst_cmd *cwr_cmd = cwrp->cwr_orig_cmd; + + TRACE_ENTRY(); + + TRACE_DBG("cwr cmd %p finished with status %d", cwr_cmd, cwr_cmd->status); + + kfree(cwrp); + + cwr_cmd->completed = 1; /* for success */ + cwr_cmd->scst_cmd_done(cwr_cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD); + + TRACE_EXIT(); + return; +} + +static void scst_cwr_write_cmd_finished(struct scst_cmd *cmd) +{ + struct scst_cwr_priv *cwrp = cmd->tgt_i_priv; + struct scst_cmd *cwr_cmd = cwrp->cwr_orig_cmd; + + TRACE_ENTRY(); + + TRACE_DBG("WRITE cmd %p finished (cwr cmd %p)", cmd, cwr_cmd); + + EXTRACHECKS_BUG_ON(cmd->cdb[0] != WRITE_16); + + if (cmd->status != 0) { + int rc; + TRACE_DBG("WRITE cmd %p (cwr cmd %p) finished not successfully", + cmd, cwr_cmd); + sBUG_ON(cmd->resp_data_len != 0); + if (cmd->status == SAM_STAT_CHECK_CONDITION) + rc = scst_set_cmd_error_sense(cwr_cmd, cmd->sense, + cmd->sense_valid_len); + else { + sBUG_ON(cmd->sense != NULL); + rc = scst_set_cmd_error_status(cwr_cmd, cmd->status); + } + if (rc != 0) { + /* Requeue possible UA */ + if (scst_is_ua_sense(cmd->sense, cmd->sense_valid_len)) + scst_requeue_ua(cmd, NULL, 0); + } + } + + scst_cwr_finished(cwrp); + + TRACE_EXIT(); + return; +} + +static void scst_cwr_read_cmd_finished(struct scst_cmd *cmd) +{ + struct scst_cwr_priv *cwrp = cmd->tgt_i_priv; + struct scst_cmd *cwr_cmd = cwrp->cwr_orig_cmd; + bool c, dif; + uint8_t write16_cdb[16]; + struct scst_cmd *wcmd; + int data_len, miscompare_offs, rc; + + TRACE_ENTRY(); + + TRACE_DBG("READ cmd %p finished (cwr cmd %p)", cmd, cwr_cmd); + + if (cmd->status != 0) { + int rc; + TRACE_DBG("Read cmd %p (cwr cmd %p) finished not successfully", + cmd, cwr_cmd); + sBUG_ON(cmd->resp_data_len != 0); + if (cmd->status == SAM_STAT_CHECK_CONDITION) + rc = scst_set_cmd_error_sense(cwr_cmd, cmd->sense, + cmd->sense_valid_len); + else { + sBUG_ON(cmd->sense != NULL); + rc = scst_set_cmd_error_status(cwr_cmd, cmd->status); + } + if (rc != 0) { + /* Requeue possible UA */ + if (scst_is_ua_sense(cmd->sense, cmd->sense_valid_len)) + scst_requeue_ua(cmd, NULL, 0); + } + goto out_finish; + } + + if (unlikely(test_bit(SCST_CMD_ABORTED, &cwr_cmd->cmd_flags))) { + TRACE_MGMT_DBG("cwr cmd %p aborted", cwr_cmd); + goto out_finish; + } + + c = sg_cmp(cmd->sg, cwr_cmd->sg, 0, 0, &miscompare_offs +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) + , KM_USER0, KM_USER1 +#endif + ); + if (!c) { + scst_set_cmd_error_and_inf(cwr_cmd, + SCST_LOAD_SENSE(scst_sense_miscompare_error), miscompare_offs); + goto out_finish; + } + + data_len = cwr_cmd->data_len; + + /* We ignore the read PI checks as required by SBC */ + rc = scst_dif_process_write(cmd); + if (unlikely(rc != 0)) + goto out_finish; + + memset(write16_cdb, 0, sizeof(write16_cdb)); + write16_cdb[0] = WRITE_16; + write16_cdb[1] = cwr_cmd->cdb[1]; + put_unaligned_be64(cwr_cmd->lba, &write16_cdb[2]); + put_unaligned_be32(data_len >> cmd->dev->block_shift, &write16_cdb[10]); + + dif = scst_cmd_needs_dif_buf(cwr_cmd) && ((cwr_cmd->cdb[1] & 0xE0) != 0); + if (dif) + write16_cdb[1] |= cwr_cmd->cdb[1] & 0xE0; + + wcmd = scst_create_prepare_internal_cmd(cwr_cmd, write16_cdb, + sizeof(write16_cdb), SCST_CMD_QUEUE_HEAD_OF_QUEUE); + if (wcmd == NULL) + goto out_busy; + + wcmd->expected_data_direction = SCST_DATA_WRITE; + wcmd->expected_transfer_len_full = data_len; + if (dif) + wcmd->expected_transfer_len_full += + data_len >> (cmd->dev->block_shift - SCST_DIF_TAG_SHIFT); + wcmd->expected_values_set = 1; + + wcmd->tgt_i_priv = cwrp; + + rc = scst_adjust_sg_get_tail(cwr_cmd, &wcmd->tgt_i_sg, + &wcmd->tgt_i_sg_cnt, &wcmd->tgt_i_dif_sg, + &wcmd->tgt_i_dif_sg_cnt, data_len, data_len); + /* + * It must not happen, because get_cdb_info_compare_and_write() + * supposed to ensure that. + */ + EXTRACHECKS_BUG_ON(rc != 0); + + wcmd->tgt_i_data_buf_alloced = 1; + + cwrp->cwr_finish_fn = scst_cwr_write_cmd_finished; + + TRACE_DBG("Adding WRITE(16) wcmd %p to head of active cmd list", wcmd); + spin_lock_irq(&wcmd->cmd_threads->cmd_list_lock); + list_add(&wcmd->cmd_list_entry, &wcmd->cmd_threads->active_cmd_list); + wake_up(&wcmd->cmd_threads->cmd_list_waitQ); + spin_unlock_irq(&wcmd->cmd_threads->cmd_list_lock); + +out: + TRACE_EXIT(); + return; + +out_busy: + scst_set_busy(cwr_cmd); + +out_finish: + scst_cwr_finished(cwrp); + goto out; +} + +int scst_cmp_wr_local(struct scst_cmd *cmd) +{ + int res = SCST_EXEC_COMPLETED; + struct scst_cwr_priv *cwrp; + uint8_t read16_cdb[16]; + struct scst_cmd *rcmd; + int data_len; + + TRACE_ENTRY(); + + /* COMPARE AND WRITE is SBC only command */ + EXTRACHECKS_BUG_ON(cmd->dev->type != TYPE_DISK); + + cmd->status = 0; + cmd->msg_status = 0; + cmd->host_status = DID_OK; + cmd->driver_status = 0; + + if (unlikely(cmd->bufflen == 0)) { + TRACE(TRACE_MINOR, "Zero bufflen (cmd %p)", cmd); + goto out_done; + } + + /* ToDo: HWALIGN'ed kmem_cache */ + cwrp = kzalloc(sizeof(*cwrp), GFP_KERNEL); + if (cwrp == NULL) { + PRINT_ERROR("Unable to allocate cwr_priv (size %zd, cmd %p)", + sizeof(*cwrp), cmd); + goto out_busy; + } + + cwrp->cwr_orig_cmd = cmd; + cwrp->cwr_finish_fn = scst_cwr_read_cmd_finished; + + data_len = cmd->data_len; + + /* + * As required by SBC, DIF PI, if any, is not checked for the read part + */ + + memset(read16_cdb, 0, sizeof(read16_cdb)); + read16_cdb[0] = READ_16; + read16_cdb[1] = cmd->cdb[1] & ~0xE0; /* as required, see above */ + put_unaligned_be64(cmd->lba, &read16_cdb[2]); + put_unaligned_be32(data_len >> cmd->dev->block_shift, &read16_cdb[10]); + + rcmd = scst_create_prepare_internal_cmd(cmd, read16_cdb, + sizeof(read16_cdb), SCST_CMD_QUEUE_HEAD_OF_QUEUE); + if (rcmd == NULL) { + res = -ENOMEM; + goto out_free; + } + + rcmd->expected_data_direction = SCST_DATA_READ; + rcmd->expected_transfer_len_full = data_len; + rcmd->expected_values_set = 1; + + rcmd->tgt_i_priv = cwrp; + + TRACE_DBG("Adding READ(16) cmd %p to head of active cmd list", rcmd); + spin_lock_irq(&rcmd->cmd_threads->cmd_list_lock); + list_add(&rcmd->cmd_list_entry, &rcmd->cmd_threads->active_cmd_list); + wake_up(&rcmd->cmd_threads->cmd_list_waitQ); + spin_unlock_irq(&rcmd->cmd_threads->cmd_list_lock); + +out: + TRACE_EXIT_RES(res); + return res; + +out_free: + kfree(cwrp); + +out_busy: + scst_set_busy(cmd); + +out_done: + cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD); + goto out; +} + + int scst_finish_internal_cmd(struct scst_cmd *cmd) { int res; @@ -6667,7 +7069,8 @@ void scst_free_cmd(struct scst_cmd *cmd) if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) TRACE_MGMT_DBG("Freeing aborted cmd %p", cmd); - EXTRACHECKS_BUG_ON(cmd->unblock_dev || cmd->dec_on_dev_needed); + EXTRACHECKS_BUG_ON(cmd->unblock_dev || cmd->dec_on_dev_needed || + cmd->on_dev_exec_list); /* * Target driver can already free sg buffer before calling @@ -7703,6 +8106,243 @@ EXPORT_SYMBOL(scst_scsi_exec_async); #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30) */ +/* + * Can switch to the next dst_sg element, so, to cmp to strictly only + * one dst_sg element, it must be either last in the chain, or + * cmp_len == dst_sg->length. + */ +static int sg_cmp_elem(struct scatterlist **pdst_sg, size_t *pdst_len, + size_t *pdst_offs, struct scatterlist *src_sg, + size_t cmp_len, int *miscompare_offs, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) + enum km_type d_km_type, enum km_type s_km_type, +#endif + bool *cmp_res) +{ + int res = 0; + struct scatterlist *dst_sg; + size_t src_len, dst_len, src_offs, dst_offs; + struct page *src_page, *dst_page; + void *saddr, *daddr; + + *cmp_res = true; + + dst_sg = *pdst_sg; + dst_len = *pdst_len; + dst_offs = *pdst_offs; + dst_page = sg_page(dst_sg); + + src_page = sg_page(src_sg); + src_len = src_sg->length; + src_offs = src_sg->offset; + + do { + size_t n; + int rc; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) + saddr = kmap_atomic(src_page + (src_offs >> PAGE_SHIFT), s_km_type) + + (src_offs & ~PAGE_MASK); + daddr = kmap_atomic(dst_page + (dst_offs >> PAGE_SHIFT), d_km_type) + + (dst_offs & ~PAGE_MASK); +#else + saddr = kmap_atomic(src_page + (src_offs >> PAGE_SHIFT)) + + (src_offs & ~PAGE_MASK); + daddr = kmap_atomic(dst_page + (dst_offs >> PAGE_SHIFT)) + + (dst_offs & ~PAGE_MASK); +#endif + + if (((src_offs & ~PAGE_MASK) == 0) && + ((dst_offs & ~PAGE_MASK) == 0) && + (src_len >= PAGE_SIZE) && (dst_len >= PAGE_SIZE) && + (cmp_len >= PAGE_SIZE)) + n = PAGE_SIZE; + else { + n = min_t(size_t, PAGE_SIZE - (dst_offs & ~PAGE_MASK), + PAGE_SIZE - (src_offs & ~PAGE_MASK)); + n = min(n, src_len); + n = min(n, dst_len); + n = min_t(size_t, n, cmp_len); + } + + /* Optimized for rare miscompares */ + + rc = memcmp(daddr, saddr, n); + if (rc != 0) { + *cmp_res = false; + if (miscompare_offs != NULL) { + int i, len = n; + unsigned long saddr_int = (unsigned long)saddr; + unsigned long daddr_int = (unsigned long)daddr; + int align_size = sizeof(unsigned long long); + int align_mask = align_size-1; + + *miscompare_offs = -1; + + if (((saddr_int & align_mask) == 0) && + ((daddr_int & align_mask) == 0) && + ((len & align_mask) == 0)) { + /* Fast path: both buffers and len aligned */ + unsigned long long *s = saddr; + unsigned long long *d = daddr; + EXTRACHECKS_BUG_ON(len % align_size != 0); + len /= align_size; + for (i = 0; i < len; i++) { + if (s[i] != d[i]) { + uint8_t *s8 = (uint8_t *)&s[i]; + uint8_t *d8 = (uint8_t *)&d[i]; + int j; + for (j = 0; j < align_size; j++) { + if (s8[j] != d8[j]) { + *miscompare_offs = i * align_size + j; + break; + } + } + break; + } + } + } else { + uint8_t *s = saddr; + uint8_t *d = daddr; + for (i = 0; i < len; i++) { + if (s[i] != d[i]) { + *miscompare_offs = i; + break; + } + } + } + EXTRACHECKS_BUG_ON(*miscompare_offs == -1); + } + goto out_unmap; + } + + dst_offs += n; + src_offs += n; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) + kunmap_atomic(saddr, s_km_type); + kunmap_atomic(daddr, d_km_type); +#else + kunmap_atomic(saddr); + kunmap_atomic(daddr); +#endif + + res += n; + cmp_len -= n; + if (cmp_len == 0) + goto out; + + src_len -= n; + dst_len -= n; + if (dst_len == 0) { + dst_sg = sg_next_inline(dst_sg); + if (dst_sg == NULL) + goto out; + dst_page = sg_page(dst_sg); + dst_len = dst_sg->length; + dst_offs = dst_sg->offset; + } + } while (src_len > 0); + +out: + *pdst_sg = dst_sg; + *pdst_len = dst_len; + *pdst_offs = dst_offs; + return res; + +out_unmap: +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) + kunmap_atomic(saddr, s_km_type); + kunmap_atomic(daddr, d_km_type); +#else + kunmap_atomic(saddr); + kunmap_atomic(daddr); +#endif + goto out; +} + +/** + * sg_cmp - compare 2 SG vectors + * @dst_sg: SG 1 + * @src_sg: SG 2 + * @nents_to_cmp: maximum number of entries to compare + * @cmp_len: maximum amount of data to compare. If 0, then compare all. + * @miscompare_offs: offset of the first miscompare. Can be NULL. + * @d_km_type: kmap_atomic type for SG 1 + * @s_km_type: kmap_atomic type for SG 2 + * + * Description: + * Data from the first SG vector will be compired with the second SG + * vector. End of the vectors will be determined by sg_next() returning + * NULL. Returns true if both vectors have identical data, false otherwise. + * + * Note! It ignores size of the vectors, so SGs with different size with + * the same data in min(sg1_size, sg2_size) size will match! + */ +static bool sg_cmp(struct scatterlist *dst_sg, struct scatterlist *src_sg, + int nents_to_cmp, size_t cmp_len, int *miscompare_offs +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) + , enum km_type d_km_type, enum km_type s_km_type +#endif + ) +{ + bool res = true; + size_t dst_len, dst_offs; + int compared_all = 0; + + TRACE_ENTRY(); + + TRACE_DBG("dst_sg %p, src_sg %p, nents_to_cmp %d, cmp_len %zd", + dst_sg, src_sg, nents_to_cmp, cmp_len); + + if (cmp_len == 0) + cmp_len = 0x7FFFFFFF; /* cmp all */ + + if (nents_to_cmp == 0) + nents_to_cmp = 0x7FFFFFFF; /* cmp all */ + + dst_len = dst_sg->length; + dst_offs = dst_sg->offset; + + do { + int compared; + + TRACE_DBG("dst_sg %p, dst_len %zd, dst_offs %zd, src_sg %p, " + "nents_to_cmp %d, cmp_len %zd, compared_all %d", + dst_sg, dst_len, dst_offs, src_sg, nents_to_cmp, + cmp_len, compared_all); + + compared = sg_cmp_elem(&dst_sg, &dst_len, &dst_offs, + src_sg, cmp_len, miscompare_offs, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) + d_km_type, s_km_type, +#endif + &res); + if (!res) { + if (miscompare_offs != NULL) { + *miscompare_offs += compared_all; + TRACE_DBG("miscompare_offs %d", + *miscompare_offs); + } + goto out; + } + cmp_len -= compared; + compared_all += compared; + if ((cmp_len == 0) || (dst_sg == NULL)) + goto out; + + nents_to_cmp--; + if (nents_to_cmp == 0) + goto out; + + src_sg = sg_next_inline(src_sg); + } while (src_sg != NULL); + +out: + TRACE_EXIT_RES(res); + return res; +} + /** * scst_copy_sg() - copy data between the command's SGs * diff --git a/scst/src/scst_priv.h b/scst/src/scst_priv.h index 02dcd96e0..f6010f85f 100644 --- a/scst/src/scst_priv.h +++ b/scst/src/scst_priv.h @@ -808,6 +808,8 @@ void scst_unthrottle_cmd(struct scst_cmd *cmd); int scst_do_internal_parsing(struct scst_cmd *cmd); int scst_parse_descriptors(struct scst_cmd *cmd); +int scst_cmp_wr_local(struct scst_cmd *cmd); + int scst_pr_init(struct scst_device *dev); void scst_pr_cleanup(struct scst_device *dev); @@ -819,6 +821,27 @@ void scst_vfs_unlink_and_put(struct path *path); int scst_copy_file(const char *src, const char *dest); +struct scst_cmd *__scst_create_prepare_internal_cmd(const uint8_t *cdb, + unsigned int cdb_len, enum scst_cmd_queue_type queue_type, + struct scst_tgt_dev *tgt_dev, gfp_t gfp_mask, bool fantom); + +static inline bool scst_lba1_inside_lba2(int64_t lba1, + int64_t lba2, int64_t lba2_blocks) +{ + bool res; + + TRACE_DBG("lba1 %lld, lba2 %lld, lba2_blocks %lld", (long long)lba1, + (long long)lba2, (long long)lba2_blocks); + + if ((lba1 >= lba2) && (lba1 < (lba2 + lba2_blocks))) + res = true; + else + res = false; + + TRACE_EXIT_RES(res); + return res; +} + #ifdef CONFIG_SCST_DEBUG_TM extern void tm_dbg_check_released_cmds(void); extern int tm_dbg_check_cmd(struct scst_cmd *cmd); diff --git a/scst/src/scst_targ.c b/scst/src/scst_targ.c index 512311068..e2218d793 100644 --- a/scst/src/scst_targ.c +++ b/scst/src/scst_targ.c @@ -103,25 +103,296 @@ static inline void scst_schedule_tasklet(struct scst_cmd *cmd) return; } -/* No locks */ -static bool scst_check_blocked_dev(struct scst_cmd *cmd) +static bool scst_unmap_overlap(struct scst_cmd *cmd, int64_t lba2, + int64_t lba2_blocks) +{ + bool res = false; + struct scst_data_descriptor *pd = cmd->cmd_data_descriptors; + int i; + + TRACE_ENTRY(); + + if (pd == NULL) + goto out; + + for (i = 0; pd[i].sdd_blocks != 0; i++) { + struct scst_data_descriptor *d = &pd[i]; + TRACE_DBG("i %d, lba %lld, blocks %lld", i, + (long long)d->sdd_lba, (long long)d->sdd_blocks); + res = scst_lba1_inside_lba2(d->sdd_lba, lba2, lba2_blocks); + if (res) + goto out; + res = scst_lba1_inside_lba2(lba2, d->sdd_lba, d->sdd_blocks); + if (res) + goto out; + } + +out: + TRACE_EXIT_RES(res); + return res; +} + +static bool scst_cmd_overlap_cwr(struct scst_cmd *cwr_cmd, struct scst_cmd *cmd) +{ + bool res; + + TRACE_ENTRY(); + + TRACE_DBG("cwr_cmd %p, cmd %p (op %s, LBA valid %d, lba %lld, " + "len %lld)", cwr_cmd, cmd, scst_get_opcode_name(cmd), + (cmd->op_flags & SCST_LBA_NOT_VALID) == 0, + (long long)cmd->lba, (long long)cmd->data_len); + + EXTRACHECKS_BUG_ON(cwr_cmd->cdb[0] != COMPARE_AND_WRITE); + + /* + * In addition to requirements listed in "Model for uninterrupted + * sequences on LBA ranges" (SBC) VMware wants that COMPARE AND WRITE + * be atomic against RESERVEs, as well as RESERVEs be atomic against + * all COMPARE AND WRITE commands and only against them. + */ + + if (cmd->op_flags & SCST_LBA_NOT_VALID) { + switch (cmd->cdb[0]) { + case RESERVE: + case RESERVE_10: + res = true; + break; + case UNMAP: + res = scst_unmap_overlap(cmd, cwr_cmd->lba, + cwr_cmd->data_len); + break; + default: + res = false; + break; + } + goto out; + } + + /* If LBA valid, block_shift must be valid */ + EXTRACHECKS_BUG_ON(cmd->dev->block_shift <= 0); + + res = scst_lba1_inside_lba2(cwr_cmd->lba, cmd->lba, + cmd->data_len >> cmd->dev->block_shift); + if (res) + goto out; + + res = scst_lba1_inside_lba2(cmd->lba, cwr_cmd->lba, + cwr_cmd->data_len >> cwr_cmd->dev->block_shift); + +out: + TRACE_EXIT_RES(res); + return res; +} + +static bool scst_cmd_overlap_reserve(struct scst_cmd *reserve_cmd, + struct scst_cmd *cmd) +{ + bool res; + + TRACE_ENTRY(); + + TRACE_DBG("reserve_cmd %p, cmd %p (op %s, LBA valid %d, lba %lld, " + "len %lld)", reserve_cmd, cmd, scst_get_opcode_name(cmd), + (cmd->op_flags & SCST_LBA_NOT_VALID) == 0, + (long long)cmd->lba, (long long)cmd->data_len); + + EXTRACHECKS_BUG_ON((reserve_cmd->cdb[0] != RESERVE) && + (reserve_cmd->cdb[0] != RESERVE_10)); + + /* + * In addition to requirements listed in "Model for uninterrupted + * sequences on LBA ranges" (SBC) VMware wants that COMPARE AND WRITE + * be atomic against RESERVEs, as well as RESERVEs be atomic against + * all COMPARE AND WRITE commands and only against them. + */ + + if (cmd->cdb[0] == COMPARE_AND_WRITE) + res = true; + else + res = false; + + TRACE_EXIT_RES(res); + return res; +} + +static bool scst_cmd_overlap_atomic(struct scst_cmd *atomic_cmd, struct scst_cmd *cmd) +{ + bool res; + + TRACE_ENTRY(); + + TRACE_DBG("atomic_cmd %p (op %s), cmd %p (op %s)", atomic_cmd, + scst_get_opcode_name(atomic_cmd), cmd, scst_get_opcode_name(cmd)); + + EXTRACHECKS_BUG_ON((atomic_cmd->op_flags & SCST_SCSI_ATOMIC) == 0); + + switch (atomic_cmd->cdb[0]) { + case COMPARE_AND_WRITE: + res = scst_cmd_overlap_cwr(atomic_cmd, cmd); + break; + case RESERVE: + case RESERVE_10: + res = scst_cmd_overlap_reserve(atomic_cmd, cmd); + break; + default: + res = false; + break; + } + + TRACE_EXIT_RES(res); + return res; +} + +static bool scst_cmd_overlap(struct scst_cmd *chk_cmd, struct scst_cmd *cmd) +{ + bool res = false; + + TRACE_ENTRY(); + + TRACE_DBG("chk_cmd %p, cmd %p", chk_cmd, cmd); + + if ((chk_cmd->op_flags & SCST_SCSI_ATOMIC) != 0) + res = scst_cmd_overlap_atomic(chk_cmd, cmd); + else if ((cmd->op_flags & SCST_SCSI_ATOMIC) != 0) + res = scst_cmd_overlap_atomic(cmd, chk_cmd); + else + res = false; + + TRACE_EXIT_RES(res); + return res; +} + +/* + * dev_lock supposed to be held and BH disabled. Returns true if cmd blocked, + * hence stop processing it and go to the next command. + */ +static bool scst_check_scsi_atomicity(struct scst_cmd *chk_cmd) +{ + bool res = false; + struct scst_device *dev = chk_cmd->dev; + struct scst_cmd *cmd; + + TRACE_ENTRY(); + + /* + * chk_cmd isn't necessary SCSI atomic! For instance, if another SCSI + * atomic cmd is waiting blocked. + */ + + TRACE_DBG("chk_cmd %p (op %s, internal %d, lba %lld, len %lld)", + chk_cmd, scst_get_opcode_name(chk_cmd), chk_cmd->internal, + (long long)chk_cmd->lba, (long long)chk_cmd->data_len); + + list_for_each_entry(cmd, &dev->dev_exec_cmd_list, dev_exec_cmd_list_entry) { + if (chk_cmd == cmd) + continue; + if (scst_cmd_overlap(chk_cmd, cmd)) { + struct scst_cmd **p = cmd->scsi_atomic_blocked_cmds; + /* + * kmalloc() allocates by at least 32 bytes increments, + * hence krealloc() on 8 bytes increments, if not all + * that space is used, does nothing. + */ + p = krealloc(p, sizeof(*p) * (cmd->scsi_atomic_blocked_cmds_count + 1), + GFP_ATOMIC); + if (p == NULL) + goto out_busy_undo; + p[cmd->scsi_atomic_blocked_cmds_count] = chk_cmd; + cmd->scsi_atomic_blocked_cmds = p; + cmd->scsi_atomic_blocked_cmds_count++; + + chk_cmd->scsi_atomic_blockers++; + + TRACE_BLOCK("Delaying cmd %p (op %s, lba %lld, " + "len %lld, blockers %d) due to overlap with " + "cmd %p (op %s, lba %lld, len %lld, blocked " + "cmds %d)", chk_cmd, scst_get_opcode_name(chk_cmd), + (long long)chk_cmd->lba, + (long long)chk_cmd->data_len, + chk_cmd->scsi_atomic_blockers, cmd, + scst_get_opcode_name(cmd), (long long)cmd->lba, + (long long)cmd->data_len, + cmd->scsi_atomic_blocked_cmds_count); + res = true; + } + } + +out: + TRACE_EXIT_RES(res); + return res; + +out_busy_undo: + list_for_each_entry(cmd, &dev->dev_exec_cmd_list, dev_exec_cmd_list_entry) { + struct scst_cmd **p = cmd->scsi_atomic_blocked_cmds; + if ((p != NULL) && (p[cmd->scsi_atomic_blocked_cmds_count-1] == chk_cmd)) { + cmd->scsi_atomic_blocked_cmds_count--; + chk_cmd->scsi_atomic_blockers--; + } + } + sBUG_ON(chk_cmd->scsi_atomic_blockers != 0); + + scst_set_busy(chk_cmd); + scst_set_cmd_abnormal_done_state(chk_cmd); + + spin_lock_irq(&chk_cmd->cmd_threads->cmd_list_lock); + TRACE_MGMT_DBG("Adding on error chk_cmd %p back to head of active cmd " + "list", chk_cmd); + list_add(&chk_cmd->cmd_list_entry, &chk_cmd->cmd_threads->active_cmd_list); + wake_up(&chk_cmd->cmd_threads->cmd_list_waitQ); + spin_unlock_irq(&chk_cmd->cmd_threads->cmd_list_lock); + + res = false; + goto out; +} + +/* + * dev_lock supposed to be BH locked. Returns true if cmd blocked, hence stop + * processing it and go to the next command. + */ +bool scst_do_check_blocked_dev(struct scst_cmd *cmd) { bool res; struct scst_device *dev = cmd->dev; TRACE_ENTRY(); - if (unlikely(cmd->internal)) { - /* - * The original command can already block the device and must - * hold reference to it, so internal command should always pass. - */ - sBUG_ON(dev->on_dev_cmd_count == 0); - res = false; - goto out; + /* + * We want to have fairness between just unblocked previously blocked + * SCSI atomic cmds and new cmds came after them. Otherwise, the new + * cmds can bypass the SCSI atomic cmds and make them unfairly wait + * again. So, we need to always, from the beginning, have blocked SCSI + * atomic cmds on the exec list, even if they blocked, as well + * as dev's SCSI atomic cmds counter incremented. + */ + + if (likely(!cmd->on_dev_exec_list)) { + list_add_tail(&cmd->dev_exec_cmd_list_entry, &dev->dev_exec_cmd_list); + cmd->on_dev_exec_list = 1; } - spin_lock_bh(&dev->dev_lock); + /* + * After a cmd passed SCSI atomicy check, there's no need to recheck SCSI + * atomicity for this cmd in future entrances here, because then all + * future overlapping with this cmd cmds will be blocked on it. + */ + + if (unlikely(((cmd->op_flags & SCST_SCSI_ATOMIC) != 0) || + (dev->dev_scsi_atomic_cmd_active != 0)) && + !cmd->scsi_atomicity_checked) { + cmd->scsi_atomicity_checked = 1; + if ((cmd->op_flags & SCST_SCSI_ATOMIC) != 0) { + dev->dev_scsi_atomic_cmd_active++; + TRACE_DBG("cmd %p (dev %p), scsi atomic_cmd_active %d", + cmd, dev, dev->dev_scsi_atomic_cmd_active); + } + + res = scst_check_scsi_atomicity(cmd); + if (res) { + EXTRACHECKS_BUG_ON(dev->dev_scsi_atomic_cmd_active == 0); + goto out; + } + } dev->on_dev_cmd_count++; cmd->dec_on_dev_needed = 1; @@ -140,8 +411,37 @@ static bool scst_check_blocked_dev(struct scst_cmd *cmd) cmd->dec_on_dev_needed = 0; TRACE_DBG("New dec on_dev_count %d (cmd %p)", dev->on_dev_cmd_count, cmd); + goto out; } +out: + TRACE_EXIT_RES(res); + return res; +} + +/* + * No locks. Returns true if cmd blocked, hence stop processing it and go to + * the next command. + */ +static bool scst_check_blocked_dev(struct scst_cmd *cmd) +{ + bool res; + struct scst_device *dev = cmd->dev; + + TRACE_ENTRY(); + + if (unlikely(cmd->internal)) { + /* + * The original command can already block the device and must + * hold reference to it, so internal command should always pass. + */ + sBUG_ON(dev->on_dev_cmd_count == 0); + res = false; + goto out; + } + + spin_lock_bh(&dev->dev_lock); + res = scst_do_check_blocked_dev(cmd); spin_unlock_bh(&dev->dev_lock); out: @@ -149,12 +449,67 @@ out: return res; } -/* No locks */ -static void scst_check_unblock_dev(struct scst_cmd *cmd) +/* dev_lock supposed to be held and BH disabled */ +static void scst_check_unblock_scsi_atomic_cmds(struct scst_cmd *cmd) +{ + int i; + + TRACE_ENTRY(); + + EXTRACHECKS_BUG_ON(cmd->scsi_atomic_blocked_cmds_count == 0); + + for (i = 0; i < cmd->scsi_atomic_blocked_cmds_count; i++) { + struct scst_cmd *acmd = cmd->scsi_atomic_blocked_cmds[i]; + acmd->scsi_atomic_blockers--; + if (acmd->scsi_atomic_blockers == 0) { + TRACE_BLOCK("Unblocking blocked acmd %p (blocker " + "cmd %p)", acmd, cmd); + spin_lock_irq(&acmd->cmd_threads->cmd_list_lock); + if (acmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE) + list_add(&acmd->cmd_list_entry, + &acmd->cmd_threads->active_cmd_list); + else + list_add_tail(&acmd->cmd_list_entry, + &acmd->cmd_threads->active_cmd_list); + wake_up(&acmd->cmd_threads->cmd_list_waitQ); + spin_unlock_irq(&acmd->cmd_threads->cmd_list_lock); + } + } + + kfree(cmd->scsi_atomic_blocked_cmds); + cmd->scsi_atomic_blocked_cmds = NULL; + cmd->scsi_atomic_blocked_cmds_count = 0; + + TRACE_EXIT(); + return; +} + +/* dev_lock supposed to be BH locked */ +void __scst_check_unblock_dev(struct scst_cmd *cmd) { struct scst_device *dev = cmd->dev; - spin_lock_bh(&dev->dev_lock); + TRACE_ENTRY(); + + /* + * We might be called here as part of Copy Manager's check blocking + * undo, so restore all flags in the previous state to allow + * restart of this cmd. + */ + + if (likely(cmd->on_dev_exec_list)) { + list_del(&cmd->dev_exec_cmd_list_entry); + cmd->on_dev_exec_list = 0; + } + + if (unlikely((cmd->op_flags & SCST_SCSI_ATOMIC) != 0)) { + if (likely(cmd->scsi_atomicity_checked)) { + dev->dev_scsi_atomic_cmd_active--; + TRACE_DBG("cmd %p, scsi atomic_cmd_active %d", + cmd, dev->dev_scsi_atomic_cmd_active); + cmd->scsi_atomicity_checked = 0; + } + } if (likely(cmd->dec_on_dev_needed)) { dev->on_dev_cmd_count--; @@ -163,6 +518,9 @@ static void scst_check_unblock_dev(struct scst_cmd *cmd) dev->on_dev_cmd_count, cmd); } + if (unlikely(cmd->scsi_atomic_blocked_cmds != NULL)) + scst_check_unblock_scsi_atomic_cmds(cmd); + if (unlikely(cmd->unblock_dev)) { TRACE_BLOCK("cmd %p (tag %llu): unblocking dev %s", cmd, (unsigned long long int)cmd->tag, dev->virt_name); @@ -184,7 +542,22 @@ static void scst_check_unblock_dev(struct scst_cmd *cmd) } } + TRACE_EXIT(); + return; +} + +/* No locks */ +void scst_check_unblock_dev(struct scst_cmd *cmd) +{ + struct scst_device *dev = cmd->dev; + + TRACE_ENTRY(); + + spin_lock_bh(&dev->dev_lock); + __scst_check_unblock_dev(cmd); spin_unlock_bh(&dev->dev_lock); + + TRACE_EXIT(); return; } @@ -3145,6 +3518,7 @@ static scst_local_exec_fn scst_local_fns[256] = { [PERSISTENT_RESERVE_OUT] = scst_persistent_reserve_out_local, [REPORT_LUNS] = scst_report_luns_local, [REQUEST_SENSE] = scst_request_sense_local, + [COMPARE_AND_WRITE] = scst_cmp_wr_local, [MAINTENANCE_IN] = scst_maintenance_in, };