diff --git a/scst/include/scst.h b/scst/include/scst.h index 80645c437..aeaef440f 100644 --- a/scst/include/scst.h +++ b/scst/include/scst.h @@ -684,6 +684,7 @@ struct scst_acg; struct scst_acg_dev; struct scst_acn; struct scst_aen; +struct scst_ext_copy_seg_descr; struct scst_opcode_descriptor; /* @@ -1330,6 +1331,12 @@ struct scst_dev_type { */ unsigned pr_cmds_notifications:1; + /* + * Set if the Copy Manager can auto assing to devices of this + * template on their registration. + */ + unsigned auto_cm_assignment_possible:1; + /* * Called to parse CDB from the cmd and initialize * cmd->bufflen and cmd->data_direction (both - REQUIRED). @@ -1420,6 +1427,26 @@ struct scst_dev_type { */ void (*on_free_cmd)(struct scst_cmd *cmd); + /* + * Called during EXTENDED COPY command processing to let dev hander + * try to remap blocks at first. Upon finish, the dev handler supposed + * to call scst_ext_copy_remap_done(). See description of this + * function for more details. + * + * In case of error, the dev handler should set the corresponding sense + * to cmd and then call scst_ext_copy_remap_done(cmd, NULL, 0). + * + * It is highly recommended that in the normal circumstances + * scst_ext_copy_remap_done() called from another thread context, + * because otherwise there will be recursion in the segments processing. + * Hopefully, this thread context switch is natural for such + * potentially long operation. + * + * OPTIONAL + */ + void (*ext_copy_remap)(struct scst_cmd *cmd, + struct scst_ext_copy_seg_descr *descr); + /* * Called to notify dev handler that a ALUA state change is about to * be started. Can be used to close open file handlers, which might @@ -1938,6 +1965,14 @@ struct scst_session { /* Used if scst_unregister_session() called in wait mode */ struct completion *shutdown_compl; + /* + * Keep list IDs for Extended Copy commands sent over this session. + * Protected by scst_cm_lock. + */ + struct list_head sess_cm_list_id_list; + + struct delayed_work sess_cm_list_id_cleanup_work; + /* sysfs release completion */ struct completion *sess_kobj_release_cmpl; @@ -2130,6 +2165,12 @@ struct scst_cmd { /* Set if cmd is internally generated */ unsigned int internal:1; + /* Set if local events should be checked for internally generated cmd */ + unsigned int internal_check_local_events:1; + + /* Set if the blocking machinery should be bypassed for this cmd */ + unsigned int bypass_blocking:1; + /* Set if the device was blocked by scst_check_blocked_dev() */ unsigned int unblock_dev:1; @@ -2501,6 +2542,7 @@ struct scst_cmd { uint64_t restart_waiting_time, rdy_to_xfer_time; uint64_t pre_exec_time, exec_time, dev_done_time; uint64_t xmit_time; + bool exec_time_counting; #endif #ifdef CONFIG_SCST_DEBUG_TM @@ -3422,6 +3464,7 @@ extern const struct scst_opcode_descriptor scst_op_descr_stpg; extern const struct scst_opcode_descriptor scst_op_descr_send_diagnostic; extern const struct scst_opcode_descriptor scst_op_descr_inquiry; +extern const struct scst_opcode_descriptor scst_op_descr_extended_copy; extern const struct scst_opcode_descriptor scst_op_descr_tur; extern const struct scst_opcode_descriptor scst_op_descr_reserve6; extern const struct scst_opcode_descriptor scst_op_descr_release6; @@ -5602,4 +5645,32 @@ int scst_pr_set_cluster_mode(struct scst_device *dev, bool cluster_mode, int scst_pr_init_dev(struct scst_device *dev); void scst_pr_clear_dev(struct scst_device *dev); +struct scst_ext_copy_data_descr { + uint64_t src_lba; + uint64_t dst_lba; + int data_len; /* in bytes */ +}; + +struct scst_ext_copy_seg_descr { +#define SCST_EXT_COPY_SEG_DATA 0 +#if 0 /* not implemented yet */ +#define SCST_EXT_COPY_SEG_PR_REG 1 +#define SCST_EXT_COPY_SEG_PR_MOVE 2 +#endif + int type; + union { + struct { + struct scst_tgt_dev *src_tgt_dev; + struct scst_tgt_dev *dst_tgt_dev; + struct scst_ext_copy_data_descr data_descr; + }; + }; + /* Internal, don't touch! */ + int tgt_descr_offs; +}; + +void scst_ext_copy_remap_done(struct scst_cmd *ec_cmd, + struct scst_ext_copy_data_descr *dds, int dds_cnt); +int scst_ext_copy_get_cur_seg_data_len(struct scst_cmd *ec_cmd); + #endif /* __SCST_H */ diff --git a/scst/include/scst_const.h b/scst/include/scst_const.h index 88327e234..42f5b9e0c 100644 --- a/scst/include/scst_const.h +++ b/scst/include/scst_const.h @@ -230,7 +230,8 @@ enum scst_cdb_flags { #endif SCST_SERIALIZED = 0x10000, SCST_STRICTLY_SERIALIZED = 0x20000|SCST_SERIALIZED, - SCST_DESCRIPTORS_BASED = 0x40000, + SCST_CAN_GEN_3PARTY_COMMANDS = 0x40000, + SCST_DESCRIPTORS_BASED = 0x80000, SCST_SCSI_ATOMIC = 0x100000, }; @@ -305,6 +306,8 @@ static inline int scst_sense_response_code(const uint8_t *sense) #define scst_sense_set_target_pgs_failed HARDWARE_ERROR, 0x67, 0xA /* ILLEGAL_REQUEST is 5 */ +#define scst_sense_operation_in_progress ILLEGAL_REQUEST, 0x0, 0x16 +#define scst_sense_parameter_list_length_invalid ILLEGAL_REQUEST, 0x1A, 0 #define scst_sense_invalid_opcode ILLEGAL_REQUEST, 0x20, 0 #define scst_sense_block_out_range_error ILLEGAL_REQUEST, 0x21, 0 /* Don't use it directly, use scst_set_invalid_field_in_cdb() instead! */ @@ -314,6 +317,10 @@ static inline int scst_sense_response_code(const uint8_t *sense) #define scst_sense_invalid_field_in_parm_list ILLEGAL_REQUEST, 0x26, 0 #define scst_sense_parameter_value_invalid ILLEGAL_REQUEST, 0x26, 2 #define scst_sense_invalid_release ILLEGAL_REQUEST, 0x26, 4 +#define scst_sense_too_many_target_descriptors ILLEGAL_REQUEST, 0x26, 6 +#define scst_sense_unsupported_tgt_descr_type ILLEGAL_REQUEST, 0x26, 7 +#define scst_sense_unsupported_seg_descr_type ILLEGAL_REQUEST, 0x26, 9 +#define scst_sense_inline_data_length_exceeded ILLEGAL_REQUEST, 0x26, 0xB #define scst_sense_saving_params_unsup ILLEGAL_REQUEST, 0x39, 0 #define scst_sense_invalid_message ILLEGAL_REQUEST, 0x49, 0 #define scst_sense_parameter_list_length_invalid ILLEGAL_REQUEST, 0x1A, 0 @@ -421,6 +428,14 @@ static inline int scst_sense_response_code(const uint8_t *sense) #define WRITE_SAME_16 0x93 #endif +#ifndef EXTENDED_COPY +#define EXTENDED_COPY 0x83 +#endif + +#ifndef RECEIVE_COPY_RESULTS +#define RECEIVE_COPY_RESULTS 0x84 +#endif + #ifndef SYNCHRONIZE_CACHE_16 #define SYNCHRONIZE_CACHE_16 0x91 #endif @@ -590,6 +605,12 @@ enum { #define SCSI_TRANSPORTID_PROTOCOLID_ISCSI 5 #define SCSI_TRANSPORTID_PROTOCOLID_SAS 6 +/* + * SCST extension. Added, because there is no TransportID for Copy Manager + * defined anywhere in the SCSI standards. + */ +#define SCST_TRANSPORTID_PROTOCOLID_COPY_MGR 0xC + /** * enum scst_tg_state - SCSI target port group asymmetric access state. * diff --git a/scst/include/scst_user.h b/scst/include/scst_user.h index 435efb3d7..1ec529b11 100644 --- a/scst/include/scst_user.h +++ b/scst/include/scst_user.h @@ -71,10 +71,11 @@ #define UCMD_STATE_BUF_ALLOCING 2 #define UCMD_STATE_EXECING 3 #define UCMD_STATE_ON_FREEING 4 -#define UCMD_STATE_ON_FREE_SKIPPED 5 -#define UCMD_STATE_ON_CACHE_FREEING 6 -#define UCMD_STATE_TM_RECEIVED_EXECING 7 -#define UCMD_STATE_TM_DONE_EXECING 8 +#define UCMD_STATE_ON_CACHE_FREEING 5 +#define UCMD_STATE_EXT_COPY_REMAPPING 6 +#define UCMD_STATE_ON_FREE_SKIPPED 7 +#define UCMD_STATE_TM_RECEIVED_EXECING 8 +#define UCMD_STATE_TM_DONE_EXECING 9 #define UCMD_STATE_ATTACH_SESS 0x20 #define UCMD_STATE_DETACH_SESS 0x21 @@ -96,6 +97,8 @@ struct scst_user_opt { uint8_t d_sense; uint8_t has_own_order_mgmt; + + uint8_t ext_copy_remap_supported; }; struct scst_user_dev_desc { @@ -212,6 +215,19 @@ struct scst_user_tm { uint8_t cmd_sn_set; }; +struct scst_user_ext_copy_data_descr { + aligned_u64 src_lba; + aligned_u64 dst_lba; + int32_t data_len; /* in bytes */ +}; + +struct scst_user_ext_copy_remap { + aligned_u64 sess_h; + aligned_u64 src_sess_h; + aligned_u64 dst_sess_h; + struct scst_user_ext_copy_data_descr data_descr; +}; + struct scst_user_get_cmd { uint32_t cmd_h; uint32_t subcode; @@ -224,6 +240,7 @@ struct scst_user_get_cmd { struct scst_user_scsi_on_free_cmd on_free_cmd; struct scst_user_on_cached_mem_free on_cached_mem_free; struct scst_user_tm tm_cmd; + struct scst_user_ext_copy_remap remap_cmd; }; }; @@ -285,6 +302,15 @@ struct scst_user_scsi_cmd_reply_exec { }; }; +/* Be careful adding new members here, this structure is allocated on stack! */ +struct scst_user_ext_copy_reply_remap { + aligned_u64 remap_descriptors; + uint16_t remap_descriptors_len; + uint8_t status; + uint8_t sense_len; + aligned_u64 psense_buffer; +}; + /* Be careful adding new members here, this structure is allocated on stack! */ struct scst_user_reply_cmd { uint32_t cmd_h; @@ -294,6 +320,7 @@ struct scst_user_reply_cmd { struct scst_user_scsi_cmd_reply_parse parse_reply; struct scst_user_scsi_cmd_reply_alloc_mem alloc_reply; struct scst_user_scsi_cmd_reply_exec exec_reply; + struct scst_user_ext_copy_reply_remap remap_reply; }; }; @@ -362,5 +389,7 @@ struct scst_user_get_multi { _IOWR('s', UCMD_STATE_TM_RECEIVED_EXECING, struct scst_user_tm) #define SCST_USER_TASK_MGMT_DONE \ _IOWR('s', UCMD_STATE_TM_DONE_EXECING, struct scst_user_tm) +#define SCST_USER_EXT_COPY_REMAP \ + _IOWR('s', UCMD_STATE_EXT_COPY_REMAPPING, struct scst_user_ext_copy_remap) #endif /* __SCST_USER_H */ diff --git a/scst/src/Makefile b/scst/src/Makefile index 7a6604030..51d83b986 100644 --- a/scst/src/Makefile +++ b/scst/src/Makefile @@ -57,6 +57,7 @@ scst-y += scst_dlm.o endif scst-y += scst_tg.o scst-y += scst_event.o +scst-y += scst_copy_mgr.o obj-$(CONFIG_SCST) += scst.o dev_handlers/ obj-$(BUILD_DEV) += $(DEV_HANDLERS_DIR)/ diff --git a/scst/src/dev_handlers/Makefile b/scst/src/dev_handlers/Makefile index d5187a677..c59b81f88 100644 --- a/scst/src/dev_handlers/Makefile +++ b/scst/src/dev_handlers/Makefile @@ -92,6 +92,8 @@ EXTRA_CFLAGS += -DCONFIG_SCST_EXTRACHECKS #EXTRA_CFLAGS += -DCONFIG_SCST_TRACING EXTRA_CFLAGS += -DCONFIG_SCST_DEBUG -g -fno-inline -fno-inline-functions +#EXTRA_CFLAGS += -DCONFIG_DEBUG_EXT_COPY_REMAP + clean: rm -f *.o *.ko .*.cmd *.mod.c .*.d .depend Modules.symvers \ Module.symvers Module.markers modules.order diff --git a/scst/src/dev_handlers/scst_user.c b/scst/src/dev_handlers/scst_user.c index 035664925..c4c65a2a2 100644 --- a/scst/src/dev_handlers/scst_user.c +++ b/scst/src/dev_handlers/scst_user.c @@ -70,6 +70,7 @@ struct scst_user_dev { unsigned int swp:1; unsigned int d_sense:1; unsigned int has_own_order_mgmt:1; + unsigned int ext_copy_remap_supported:1; int (*generic_parse)(struct scst_cmd *cmd); @@ -949,6 +950,37 @@ static int dev_user_exec(struct scst_cmd *cmd) return res; } +static void dev_user_ext_copy_remap(struct scst_cmd *cmd, + struct scst_ext_copy_seg_descr *seg) +{ + struct scst_user_cmd *ucmd = cmd->dh_priv; + + TRACE_ENTRY(); + + TRACE_DBG("Preparing EXT_COPY_REMAP for user space (ucmd=%p, h=%d, " + "seg %p)", ucmd, ucmd->h, seg); + + ucmd->user_cmd_payload_len = + offsetof(struct scst_user_get_cmd, remap_cmd) + + sizeof(ucmd->user_cmd.remap_cmd); + ucmd->user_cmd.cmd_h = ucmd->h; + ucmd->user_cmd.subcode = SCST_USER_EXT_COPY_REMAP; + ucmd->user_cmd.remap_cmd.sess_h = (unsigned long)cmd->tgt_dev; + + ucmd->user_cmd.remap_cmd.src_sess_h = (unsigned long)seg->src_tgt_dev; + ucmd->user_cmd.remap_cmd.dst_sess_h = (unsigned long)seg->dst_tgt_dev; + ucmd->user_cmd.remap_cmd.data_descr.src_lba = seg->data_descr.src_lba; + ucmd->user_cmd.remap_cmd.data_descr.dst_lba = seg->data_descr.dst_lba; + ucmd->user_cmd.remap_cmd.data_descr.data_len = seg->data_descr.data_len; + + ucmd->state = UCMD_STATE_EXT_COPY_REMAPPING; + + dev_user_add_to_ready(ucmd); + + TRACE_EXIT(); + return; +} + static void dev_user_free_sgv(struct scst_user_cmd *ucmd) { if (ucmd->sgv != NULL) { @@ -1180,7 +1212,8 @@ static void dev_user_add_to_ready(struct scst_user_cmd *ucmd) &dev->ready_cmd_list); } do_wake |= ((ucmd->state == UCMD_STATE_ON_CACHE_FREEING) || - (ucmd->state == UCMD_STATE_ON_FREEING)); + (ucmd->state == UCMD_STATE_ON_FREEING) || + (ucmd->state == UCMD_STATE_EXT_COPY_REMAPPING)); } if (do_wake) { @@ -1444,6 +1477,150 @@ static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd) return res; } +static int dev_user_process_reply_ext_copy_remap(struct scst_user_cmd *ucmd, + struct scst_user_reply_cmd *reply) +{ + int res = 0, rc, count = 0, i, len; + struct scst_user_ext_copy_reply_remap *rreply = &reply->remap_reply; + struct scst_cmd *cmd = ucmd->cmd; + uint8_t *buf; + struct scst_user_ext_copy_data_descr *uleft; + struct scst_ext_copy_data_descr *left = NULL; + + TRACE_ENTRY(); + + if (unlikely((rreply->status != 0) || (rreply->sense_len != 0))) + goto out_status; + + if (rreply->remap_descriptors_len == 0) { + TRACE_DBG("No remap leftovers (ucmd %p, cmd %p)", ucmd, cmd); + goto out_done; + } + + if (unlikely(rreply->remap_descriptors_len > PAGE_SIZE)) { + PRINT_ERROR("Too many leftover REMAP descriptors (len %d, ucmd %p, " + "cmd %p)", rreply->remap_descriptors_len, ucmd, cmd); + res = -EOVERFLOW; + goto out_hw_err; + } + + buf = kzalloc(rreply->remap_descriptors_len, GFP_KERNEL); + if (unlikely(buf == NULL)) { + PRINT_ERROR("Unable to alloc leftover remap descriptors buf " + "(size %d)", rreply->remap_descriptors_len); + goto out_busy; + } + + rc = copy_from_user(buf, + (void __user *)(unsigned long)rreply->remap_descriptors, + rreply->remap_descriptors_len); + if (unlikely(rc != 0)) { + PRINT_ERROR("Failed to copy %d leftover remap descriptors' " + "bytes", rc); + res = -EFAULT; + goto out_free_buf; + } + + uleft = (struct scst_user_ext_copy_data_descr *)buf; + count = rreply->remap_descriptors_len / sizeof(*uleft); + if (unlikely((rreply->remap_descriptors_len % sizeof(*uleft)) != 0)) { + PRINT_ERROR("Invalid leftover remap descriptors (ucmd %p, " + "cmd %p, count %d)", ucmd, cmd, count); + res = -EINVAL; + goto out_hw_err_free_buf; + } + + left = kmalloc(sizeof(*left) * count, GFP_KERNEL); + if (unlikely(left == NULL)) { + PRINT_ERROR("Unable to alloc leftover remap descriptors " + "(size %zd, count %d)", sizeof(*left) * count, count); + goto out_busy_free_buf; + } + + TRACE_DBG("count %d", count); + + len = 0; + for (i = 0; i < count; i++) { + TRACE_DBG("src_lba %lld, dst_lba %lld, data_len %d (len %d)", + (long long)uleft[i].src_lba, (long long)uleft[i].dst_lba, + uleft[i].data_len, len); + if (unlikely(uleft[i].data_len == 0)) { + PRINT_ERROR("Invalid data descr %d len %d (cmd %p)", i, + uleft[i].data_len, cmd); + res = -EINVAL; + goto out_hw_err_free_buf; + } + left[i].src_lba = uleft[i].src_lba; + left[i].dst_lba = uleft[i].dst_lba; + left[i].data_len = uleft[i].data_len; + len += uleft[i].data_len; + } + + if (unlikely(len > scst_ext_copy_get_cur_seg_data_len(cmd))) { + PRINT_ERROR("Invalid data descr len %d (cmd %p, seg descr len %d)", + len, cmd, scst_ext_copy_get_cur_seg_data_len(cmd)); + res = -EINVAL; + goto out_hw_err_free_buf; + } + +out_free_buf: + kfree(buf); + +out_done: + if (unlikely(res != 0)) { + kfree(left); + left = NULL; + count = 0; + } + + scst_ext_copy_remap_done(cmd, left, count); + + TRACE_EXIT_RES(res); + return res; + +out_hw_err: + scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error)); + goto out_done; + +out_hw_err_free_buf: + scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error)); + goto out_free_buf; + +out_busy_free_buf: + scst_set_busy(cmd); + goto out_free_buf; + +out_busy: + scst_set_busy(cmd); + goto out_done; + +out_status: + TRACE_DBG("Remap finished with status %d (ucmd %p, cmd %p)", + rreply->status, ucmd, cmd); + + if (rreply->sense_len != 0) { + int sense_len; + + res = scst_alloc_sense(cmd, 0); + if (res != 0) + goto out_hw_err; + + sense_len = min_t(int, cmd->sense_buflen, rreply->sense_len); + + rc = copy_from_user(cmd->sense, + (void __user *)(unsigned long)rreply->psense_buffer, + sense_len); + if (rc != 0) { + PRINT_ERROR("Failed to copy %d sense's bytes", rc); + res = -EFAULT; + goto out_hw_err; + } + cmd->sense_valid_len = sense_len; + } + scst_set_cmd_error_status(cmd, rreply->status); + goto out_done; +} + static int dev_user_process_ws_reply(struct scst_user_cmd *ucmd, struct scst_user_scsi_cmd_reply_exec *ereply) { @@ -1764,6 +1941,10 @@ unlock_process: res = dev_user_process_reply_on_cache_free(ucmd); break; + case UCMD_STATE_EXT_COPY_REMAPPING: + res = dev_user_process_reply_ext_copy_remap(ucmd, reply); + break; + case UCMD_STATE_TM_RECEIVED_EXECING: case UCMD_STATE_TM_DONE_EXECING: res = dev_user_process_reply_tm_exec(ucmd, @@ -2460,6 +2641,7 @@ static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy, break; case UCMD_STATE_EXECING: + case UCMD_STATE_EXT_COPY_REMAPPING: if (flags != NULL) spin_unlock_irqrestore(&dev->udev_cmd_threads.cmd_list_lock, *flags); @@ -2478,8 +2660,13 @@ static void dev_user_unjam_cmd(struct scst_user_cmd *ucmd, int busy, SCST_LOAD_SENSE(scst_sense_lun_not_supported)); } - ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT, - SCST_CONTEXT_THREAD); + if (state == UCMD_STATE_EXECING) + ucmd->cmd->scst_cmd_done(ucmd->cmd, SCST_CMD_STATE_DEFAULT, + SCST_CONTEXT_THREAD); + else { + sBUG_ON(state != UCMD_STATE_EXT_COPY_REMAPPING); + scst_ext_copy_remap_done(ucmd->cmd, NULL, 0); + } /* !! At this point cmd and ucmd can be already freed !! */ if (flags != NULL) @@ -3018,6 +3205,11 @@ static void dev_user_setup_functions(struct scst_user_dev *dev) dev->devtype.dev_alloc_data_buf = dev_user_alloc_data_buf; dev->devtype.dev_done = NULL; + if (dev->ext_copy_remap_supported) + dev->devtype.ext_copy_remap = dev_user_ext_copy_remap; + else + dev->devtype.ext_copy_remap = NULL; + if (dev->parse_type != SCST_USER_PARSE_CALL) { switch (dev->devtype.type) { case TYPE_DISK: @@ -3471,9 +3663,10 @@ static int __dev_user_set_opt(struct scst_user_dev *dev, TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, " "memory_reuse_type %x, partial_transfers_type %x, " - "partial_len %d", dev->name, opt->parse_type, - opt->on_free_cmd_type, opt->memory_reuse_type, - opt->partial_transfers_type, opt->partial_len); + "partial_len %d, opt->ext_copy_remap_supported %d", + dev->name, opt->parse_type, opt->on_free_cmd_type, + opt->memory_reuse_type, opt->partial_transfers_type, + opt->partial_len, opt->ext_copy_remap_supported); if (opt->parse_type > SCST_USER_MAX_PARSE_OPT || opt->on_free_cmd_type > SCST_USER_MAX_ON_FREE_CMD_OPT || @@ -3492,12 +3685,13 @@ static int __dev_user_set_opt(struct scst_user_dev *dev, ((opt->qerr == SCST_QERR_2_RESERVED) || (opt->qerr > SCST_QERR_3_ABORT_THIS_NEXUS_ONLY)) || (opt->swp > 1) || (opt->tas > 1) || (opt->has_own_order_mgmt > 1) || - (opt->d_sense > 1)) { + (opt->d_sense > 1) || (opt->ext_copy_remap_supported > 1)) { PRINT_ERROR("Invalid SCSI option (tst %x, tmf_only %x, " "queue_alg %x, qerr %x, swp %x, tas %x, d_sense %d, " - "has_own_order_mgmt %x)", + "has_own_order_mgmt %x, ext_copy_remap_supported %d)", opt->tst, opt->tmf_only, opt->queue_alg, opt->qerr, - opt->swp, opt->tas, opt->d_sense, opt->has_own_order_mgmt); + opt->swp, opt->tas, opt->d_sense, opt->has_own_order_mgmt, + opt->ext_copy_remap_supported); res = -EINVAL; goto out; } @@ -3526,6 +3720,7 @@ static int __dev_user_set_opt(struct scst_user_dev *dev, dev->tas = opt->tas; dev->d_sense = opt->d_sense; dev->has_own_order_mgmt = opt->has_own_order_mgmt; + dev->ext_copy_remap_supported = opt->ext_copy_remap_supported; if (dev->sdev != NULL) { dev->sdev->tst = opt->tst; dev->sdev->tmf_only = opt->tmf_only; @@ -3595,12 +3790,14 @@ static int dev_user_get_opt(struct file *file, void __user *arg) opt.swp = dev->swp; opt.d_sense = dev->d_sense; opt.has_own_order_mgmt = dev->has_own_order_mgmt; + opt.ext_copy_remap_supported = dev->ext_copy_remap_supported; TRACE_DBG("dev %s, parse_type %x, on_free_cmd_type %x, " "memory_reuse_type %x, partial_transfers_type %x, " - "partial_len %d", dev->name, opt.parse_type, - opt.on_free_cmd_type, opt.memory_reuse_type, - opt.partial_transfers_type, opt.partial_len); + "partial_len %d, ext_copy_remap_supported %d", dev->name, + opt.parse_type, opt.on_free_cmd_type, opt.memory_reuse_type, + opt.partial_transfers_type, opt.partial_len, + opt.ext_copy_remap_supported); rc = copy_to_user(arg, &opt, sizeof(opt)); if (unlikely(rc != 0)) { diff --git a/scst/src/dev_handlers/scst_vdisk.c b/scst/src/dev_handlers/scst_vdisk.c index 8271555f8..e47d79631 100644 --- a/scst/src/dev_handlers/scst_vdisk.c +++ b/scst/src/dev_handlers/scst_vdisk.c @@ -357,6 +357,10 @@ static ssize_t vcdrom_del_device(const char *device_name); static void vdisk_task_mgmt_fn_done(struct scst_mgmt_cmd *mcmd, struct scst_tgt_dev *tgt_dev); static uint64_t vdisk_gen_dev_id_num(const char *virt_dev_name); +#ifdef CONFIG_DEBUG_EXT_COPY_REMAP +static void vdev_ext_copy_remap(struct scst_cmd *cmd, + struct scst_ext_copy_seg_descr *descr); +#endif static int vdisk_unmap_range(struct scst_cmd *cmd, struct scst_vdisk_dev *virt_dev, uint64_t start_lba, uint32_t blocks); @@ -677,6 +681,7 @@ static struct scst_dev_type vdisk_file_devtype = { .threads_num = -1, .parse_atomic = 1, .dev_done_atomic = 1, + .auto_cm_assignment_possible = 1, .attach = vdisk_attach, .detach = vdisk_detach, .attach_tgt = vdisk_attach_tgt, @@ -686,6 +691,9 @@ static struct scst_dev_type vdisk_file_devtype = { .exec = fileio_exec, .on_free_cmd = fileio_on_free_cmd, .task_mgmt_fn_done = vdisk_task_mgmt_fn_done, +#ifdef CONFIG_DEBUG_EXT_COPY_REMAP + .ext_copy_remap = vdev_ext_copy_remap, +#endif .get_supported_opcodes = vdisk_get_supported_opcodes, .devt_priv = (void *)fileio_ops, #ifdef CONFIG_SCST_PROC @@ -734,6 +742,7 @@ static struct scst_dev_type vdisk_blk_devtype = { #ifdef CONFIG_SCST_PROC .no_proc = 1, #endif + .auto_cm_assignment_possible = 1, .attach = vdisk_attach, .detach = vdisk_detach, .attach_tgt = vdisk_attach_tgt, @@ -784,6 +793,7 @@ static struct scst_dev_type vdisk_null_devtype = { #ifdef CONFIG_SCST_PROC .no_proc = 1, #endif + .auto_cm_assignment_possible = 1, .attach = vdisk_attach, .detach = vdisk_detach, .attach_tgt = vdisk_attach_tgt, @@ -827,6 +837,7 @@ static struct scst_dev_type vcdrom_devtype = { .threads_num = -1, .parse_atomic = 1, .dev_done_atomic = 1, + .auto_cm_assignment_possible = 1, .attach = vdisk_attach, .detach = vdisk_detach, .attach_tgt = vdisk_attach_tgt, @@ -2757,6 +2768,7 @@ static vdisk_op_fn nullio_ops[256] = { &scst_op_descr_write_same16, \ &scst_op_descr_unmap, \ &scst_op_descr_format_unit, \ + &scst_op_descr_extended_copy, \ &scst_op_descr_cwr, static const struct scst_opcode_descriptor *vdisk_opcode_descriptors[] = { @@ -4256,6 +4268,7 @@ static int vdisk_inq(uint8_t *buf, struct scst_cmd *cmd, if (virt_dev->expl_alua) buf[5] |= SCST_INQ_TPGS_MODE_EXPLICIT; } + buf[5] |= 8; /* 3PC */ buf[6] = 0x10; /* MultiP 1 */ buf[7] = 2; /* CMDQUE 1, BQue 0 => commands queuing supported */ @@ -7365,6 +7378,74 @@ static void vdisk_task_mgmt_fn_done(struct scst_mgmt_cmd *mcmd, return; } +#ifdef CONFIG_DEBUG_EXT_COPY_REMAP +static void vdev_ext_copy_remap(struct scst_cmd *cmd, + struct scst_ext_copy_seg_descr *seg) +{ + struct scst_ext_copy_data_descr *d; + static int shift = 0; + static DEFINE_SPINLOCK(lock); + int s; + + TRACE_ENTRY(); + + if (seg->data_descr.data_len <= 4096) { + /* No way to split */ + goto out_done; + } + + d = kzalloc(sizeof(*d)*2, GFP_KERNEL); + if (d == NULL) + goto out_busy; + + spin_lock(&lock); + + shift += 4096; + + if (shift >= seg->data_descr.data_len) { + shift = 0; + s = 0; + } else + s = shift; + + TRACE_DBG("cmd %p, seg %p, data_len %d, shift %d, s %d", cmd, seg, + seg->data_descr.data_len, shift, s); + + spin_unlock(&lock); + + if (s == 0) + goto out_free_done; + + d[0].data_len = s; + d[0].src_lba = seg->data_descr.src_lba; + d[0].dst_lba = seg->data_descr.dst_lba; + + d[1].data_len = seg->data_descr.data_len - s; + d[1].src_lba = seg->data_descr.src_lba + (s >> seg->src_tgt_dev->dev->block_shift); + d[1].dst_lba = seg->data_descr.dst_lba + (s >> seg->dst_tgt_dev->dev->block_shift); + + scst_ext_copy_remap_done(cmd, d, 2); + +out: + TRACE_EXIT(); + return; + +out_busy: + scst_set_busy(cmd); + +out_free_done: + kfree(d); + +out_done: +#if 1 + scst_ext_copy_remap_done(cmd, &seg->data_descr, 1); +#else + scst_ext_copy_remap_done(cmd, NULL, 0); +#endif + goto out; +} +#endif + static void vdisk_report_registering(const struct scst_vdisk_dev *virt_dev) { enum { buf_size = 256 }; diff --git a/scst/src/scst_copy_mgr.c b/scst/src/scst_copy_mgr.c new file mode 100644 index 000000000..63183f206 --- /dev/null +++ b/scst/src/scst_copy_mgr.c @@ -0,0 +1,3741 @@ +/* + * Copyright (C) 2013 - 2015 SanDisk Corporation + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "scst.h" +#include "scst_priv.h" +#include "scst_pres.h" + +#ifdef CONFIG_SCST_PROC +#warning In the PROCFS build EXTENDED COPY not supported +#else + +#define SCST_CM_NAME "copy_manager" +#define SCST_CM_TGT_NAME (SCST_CM_NAME "_tgt") +#define SCST_CM_SESS_NAME (SCST_CM_NAME "_sess") + +#define SCST_CM_TID_SIZE 24 +#define SCST_CM_TID_ID "COPY_MGR" + +#define SCST_CM_RETRIES_WAIT HZ +#define SCST_CM_MAX_RETRIES_TIME (30*HZ) +#define SCST_CM_ID_KEEP_TIME (5*HZ) + +#define SCST_CM_MAX_EACH_IO_SIZE (512*1024) + +/* Too big value is not too good for the blocking machinery */ +#define SCST_CM_MAX_TGT_DESCR_CNT 5 + +static struct scst_tgt *scst_cm_tgt; +static struct scst_session *scst_cm_sess; + +/* Protected by scst_mutex */ +static unsigned int scst_cm_next_lun; + +static DEFINE_MUTEX(scst_cm_mutex); + +/* Protected by scst_cm_mutex */ +static LIST_HEAD(scst_cm_desig_list); + +struct scst_cm_desig { + struct list_head cm_desig_list_entry; + + struct scst_tgt_dev *desig_tgt_dev; + + int desig_len; + uint8_t desig[]; +}; + +/* It's IRQ and inner for sess_list_lock */ +static spinlock_t scst_cm_lock; + +/* Necessary fields protected by scst_cm_lock */ +struct scst_cm_list_id { + struct list_head sess_cm_list_id_entry; + + int cm_lid; + +#define SCST_CM_LIST_ID_STATE_ACTIVE 0 +#define SCST_CM_LIST_ID_STATE_DONE 1 +#define SCST_CM_LIST_ID_STATE_PENDING_FREE 2 + int cm_list_id_state; + + unsigned int cm_done:1; + unsigned int cm_can_be_immed_free:1; + + int cm_segs_processed; + int64_t cm_written_size; /* in bytes */ + + unsigned long cm_time_to_free; /* in jiffies */ + + int cm_status; + unsigned short cm_sense_len; + uint8_t cm_sense[SCST_SENSE_BUFFERSIZE]; +}; + +struct scst_cm_internal_cmd_priv { + /* Must be the first for scst_finish_internal_cmd()! */ + scst_i_finish_fn_t cm_finish_fn; + + /* Internal cmd itself. Needed for unified cleanups and aborts. */ + struct scst_cmd *cm_cmd; + + /* E.g. rcmd for WRITEs, ec_cmd for READs, etc. */ + struct scst_cmd *cm_orig_cmd; + + struct list_head cm_internal_cmd_list_entry; +}; + +struct scst_cm_dev_entry { + struct list_head cm_sorted_devs_list_entry; + struct scst_cmd *cm_fcmd; /* can point to the real EC cmd as well! */ +}; + +/* + * Most fields here not protected, since can be accessed only from a single + * thread at time. + */ +struct scst_cm_ec_cmd_priv { + struct scst_cm_list_id *cm_list_id; + + /* + * List of all devices involved in this EC cmd (structs + * scst_cm_dev_entry), sorted by their dev's address. + */ + struct list_head cm_sorted_devs_list; + + /* + * List of all generated internal commands. Where needed, protected + * by scst_cm_lock. + */ + struct list_head cm_internal_cmd_list; + +#define SCST_CM_ERROR_NONE 0 +#define SCST_CM_ERROR_READ 1 +#define SCST_CM_ERROR_WRITE 2 + int cm_error; + + struct mutex cm_mutex; + + int cm_cur_in_flight; /* commands */ + + /** + ** READ commands stuff + **/ + struct scst_tgt_dev *cm_read_tgt_dev; + int64_t cm_start_read_lba; /* in blocks */ + int64_t cm_cur_read_lba; /* in blocks */ + int cm_left_to_read; /* in blocks */ + int cm_max_each_read;/* in blocks */ + + /** + ** WRITE commands stuff + **/ + struct scst_tgt_dev *cm_write_tgt_dev; + int64_t cm_start_write_lba; /* in blocks */ + int64_t cm_written; /* in bytes */ + + /** + ** Current data descriptors and their count + **/ + const struct scst_ext_copy_data_descr *cm_data_descrs; + int cm_data_descrs_cnt; + int cm_cur_data_descr; + + int cm_cur_seg_descr; + + /** + ** Parsed descriptors. Number of them is in + ** cmd->cmd_data_descriptors_cnt + **/ + struct scst_ext_copy_seg_descr cm_seg_descrs[]; +}; + +#define SCST_ALLOW_NOT_CONN_COPY_DEF 0 +/* Not protected, because no need */ +static bool scst_cm_allow_not_connected_copy = SCST_ALLOW_NOT_CONN_COPY_DEF; + +#define SCST_CM_STATUS_CMD_SUCCEEDED 0 +#define SCST_CM_STATUS_RETRY 1 +#define SCST_CM_STATUS_CMD_FAILED -1 + +typedef void (*scst_cm_retry_fn_t)(struct scst_cmd *cmd); + +struct scst_cm_retry { + struct scst_cmd *cm_retry_cmd; + struct delayed_work cm_retry_work; + scst_cm_retry_fn_t cm_retry_fn; +}; + +static void scst_cm_retry_work_fn(struct delayed_work *work) +{ + struct scst_cm_retry *retry = container_of(work, struct scst_cm_retry, + cm_retry_work); + + TRACE_ENTRY(); + + TRACE_DBG("Retrying cmd %p", retry->cm_retry_cmd); + + retry->cm_retry_fn(retry->cm_retry_cmd); + + __scst_cmd_put(retry->cm_retry_cmd); + + kfree(retry); + + TRACE_EXIT(); + return; +} + +/* + * Checks if cmd finished successfully and performs/schedules retry, if necessary. + * Returns one of SCST_CM_STATUS_* codes. + */ +static int scst_cm_err_check_retry(struct scst_cmd *cmd, + unsigned long start_time, scst_cm_retry_fn_t retry_fn) +{ + int res = SCST_CM_STATUS_CMD_SUCCEEDED; + unsigned long cur_time, max_retry_time, next_retry_time; + struct scst_cm_retry *retry; + bool imm_retry = false; + + TRACE_ENTRY(); + + /* cmd->dev and tgt_dev can be NULL here! */ + + TRACE_DBG("cmd %p, status %d, aborted %d", cmd, cmd->status, + scst_cmd_aborted(cmd)); + + if (likely((cmd->status == 0) && !scst_cmd_aborted(cmd))) + goto out; + + cur_time = jiffies; + max_retry_time = start_time + SCST_CM_MAX_RETRIES_TIME; + + mutex_lock(&scst_cm_mutex); + + if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) { + if (test_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags)) { + TRACE_MGMT_DBG("Cmd %p aborted by other initiator, " + "retry possible", cmd); + goto try_retry; + } else { + TRACE_MGMT_DBG("Cmd %p aborted, no retry ", cmd); + goto out_err_unlock; + } + } + + if ((cmd->status == SAM_STAT_BUSY) || + (cmd->status == SAM_STAT_TASK_SET_FULL) || + (cmd->status == SAM_STAT_RESERVATION_CONFLICT) || + (cmd->status == SAM_STAT_ACA_ACTIVE)) { + TRACE_DBG("Cmd %p finished with status %d, retry possible", + cmd, cmd->status); + goto try_retry; + } + + if ((cmd->status == SAM_STAT_CHECK_CONDITION) && + scst_sense_valid(cmd->sense) && + scst_analyze_sense(cmd->sense, cmd->sense_valid_len, + SCST_SENSE_KEY_VALID, UNIT_ATTENTION, 0, 0)) { + TRACE_DBG("Cmd %p finished with UA, immediate retry " + "possible", cmd); + imm_retry = true; + goto try_retry; + } + +out_err_unlock: + mutex_unlock(&scst_cm_mutex); + +out_failed: + res = SCST_CM_STATUS_CMD_FAILED; + +out: + TRACE_EXIT_RES(res); + return res; + +try_retry: + if (time_after_eq(cur_time, max_retry_time)) + goto out_err_unlock; + + next_retry_time = cur_time + SCST_CM_RETRIES_WAIT; + + TRACE_DBG("Retrying cmd %p (imm_retry %d, next_retry_time %ld, " + "cur_time %ld, " "start_time %ld, max_retry_time %ld): going " + "to sleep", cmd, imm_retry, next_retry_time, cur_time, + start_time, max_retry_time); + + mutex_unlock(&scst_cm_mutex); + + if (retry_fn == NULL) + goto out_retry_done; + + /* Wait before retry */ + + retry = kzalloc(sizeof(*retry), GFP_KERNEL); + if (retry == NULL) { + PRINT_ERROR("Unable to allocate retry struct"); + scst_set_busy(cmd); + goto out_failed; + } + retry->cm_retry_cmd = cmd; + __scst_cmd_get(cmd); + INIT_DELAYED_WORK(&retry->cm_retry_work, + (void (*)(struct work_struct *))scst_cm_retry_work_fn); + retry->cm_retry_fn = retry_fn; + + if (imm_retry) { + /* Let's use work to avoid possible recursion */ + TRACE_DBG("Immediate retry (cmd %p)", cmd); + schedule_work(&retry->cm_retry_work.work); + } else { + TRACE_DBG("Scheduling cmd %p retry", cmd); + schedule_delayed_work(&retry->cm_retry_work, + next_retry_time - cur_time); + } + +out_retry_done: + res = SCST_CM_STATUS_RETRY; + goto out; +} + +static bool scst_cm_is_ec_cmd_done(struct scst_cmd *ec_cmd) +{ + bool res; + + TRACE_ENTRY(); + + if (unlikely(test_bit(SCST_CMD_ABORTED, &ec_cmd->cmd_flags))) { + TRACE_MGMT_DBG("EC cmd %p aborted", ec_cmd); + res = true; + } else if (unlikely(ec_cmd->completed)) { + TRACE_MGMT_DBG("EC cmd %p already completed with status (%d)", + ec_cmd, ec_cmd->status); + res = true; + } else + res = false; + + TRACE_EXIT_RES(res); + return res; +} + +/* + * cm_mutex suppose to be locked or no activities on this ec_cmd's priv. + * + * Returns 0 on success or -ENOENT, if there's no more data descriptors in + * ec_cmd, or other negative error code. For other error codes cmd status + * and sense supposed to be set. + */ +static int scst_cm_setup_this_data_descr(struct scst_cmd *ec_cmd) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + struct scst_ext_copy_seg_descr *sd; + const struct scst_ext_copy_data_descr *dd; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p, cm_cur_data_descr %d", ec_cmd, + priv->cm_cur_data_descr); + + EXTRACHECKS_BUG_ON(priv->cm_cur_data_descr > priv->cm_data_descrs_cnt); + + if (priv->cm_cur_data_descr == priv->cm_data_descrs_cnt) { + TRACE_DBG("No more data desriptors for ec_cmd %p", ec_cmd); + res = -ENOENT; + goto out; + } + + sd = &priv->cm_seg_descrs[priv->cm_cur_seg_descr]; + dd = &priv->cm_data_descrs[priv->cm_cur_data_descr]; + + EXTRACHECKS_BUG_ON(dd->data_len == 0); + + priv->cm_read_tgt_dev = sd->src_tgt_dev; + priv->cm_start_read_lba = dd->src_lba; + priv->cm_cur_read_lba = dd->src_lba; + priv->cm_left_to_read = dd->data_len >> sd->src_tgt_dev->dev->block_shift; + priv->cm_max_each_read = SCST_CM_MAX_EACH_IO_SIZE >> sd->src_tgt_dev->dev->block_shift; + + priv->cm_write_tgt_dev = sd->dst_tgt_dev; + priv->cm_start_write_lba = dd->dst_lba; + + TRACE_DBG("len %d, src_lba %lld, dst_lba %lld", dd->data_len, + (long long)dd->src_lba, (long long)dd->dst_lba); + + if (unlikely((dd->data_len & (sd->src_tgt_dev->dev->block_size-1)) != 0) || + unlikely((dd->data_len & (sd->dst_tgt_dev->dev->block_size-1)) != 0)) { + PRINT_ERROR("Data len %d is not even for block size (src block " + "size %d, dst block size %d)", dd->data_len, + sd->src_tgt_dev->dev->block_size, + sd->dst_tgt_dev->dev->block_size); + scst_set_cmd_error(ec_cmd, SCST_LOAD_SENSE(scst_sense_hardw_error)); + res = -EINVAL; + goto out; + } + + res = 0; + +out: + TRACE_EXIT_RES(res); + return res; +} + +/* + * cm_mutex suppose to be locked or no activities on this ec_cmd's priv. + * + * Returns 0 on success or -ENOENT, if there's no more data descriptors in + * ec_cmd, or other negative error code. For other error codes cmd status + * and sense supposed to be set. + */ +static int scst_cm_setup_next_data_descr(struct scst_cmd *ec_cmd) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p", ec_cmd); + + priv->cm_cur_data_descr++; + + res = scst_cm_setup_this_data_descr(ec_cmd); + + TRACE_EXIT_RES(res); + return res; +} + +/* + * cm_mutex suppose to be locked or no activities on this ec_cmd's priv. + * + * Returns 0 on success or -ENOENT, if there are no data descriptors in + * ec_cmd, or other negative error code. For other error codes cmd status + * and sense supposed to be set. + */ +static int scst_cm_setup_first_data_descr(struct scst_cmd *ec_cmd) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p", ec_cmd); + + priv->cm_cur_data_descr = 0; + + res = scst_cm_setup_this_data_descr(ec_cmd); + + TRACE_EXIT_RES(res); + return res; +} + +static void scst_cm_destroy_data_descrs(struct scst_cmd *ec_cmd) +{ + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p, data_descrs %p, data_descrs_cnt %d ", ec_cmd, + priv->cm_data_descrs, priv->cm_data_descrs_cnt); + + if (priv->cm_data_descrs != &priv->cm_seg_descrs[priv->cm_cur_seg_descr].data_descr) { + TRACE_DBG_FLAG(TRACE_DEBUG|TRACE_MEMORY, "Freeing " + "data_descrs %p", priv->cm_data_descrs); + kfree(priv->cm_data_descrs); + } + + priv->cm_data_descrs = NULL; + priv->cm_data_descrs_cnt = 0; + + TRACE_EXIT(); + return; +} + +/* + * cm_mutex suppose to be locked or no activities on this ec_cmd's priv. + * + * Returns 0 on success or -ENOENT, if there are no data descriptors in + * ec_cmd, or other negative error code. For other error codes cmd status + * and sense supposed to be set. + */ +static int scst_cm_setup_data_descrs(struct scst_cmd *ec_cmd, + const struct scst_ext_copy_data_descr *dds, int dds_cnt) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p, dds %p, dds_cnt %d", ec_cmd, dds, dds_cnt); + + EXTRACHECKS_BUG_ON(dds_cnt == 0); + + priv->cm_data_descrs = dds; + priv->cm_data_descrs_cnt = dds_cnt; + + res = scst_cm_setup_first_data_descr(ec_cmd); + if (unlikely(res != 0)) + goto out_destr; + +out: + TRACE_EXIT_RES(res); + return res; + +out_destr: + scst_cm_destroy_data_descrs(ec_cmd); + goto out; +} + +/* + * cm_mutex suppose to be locked or no activities on this ec_cmd's priv. + * + * Returns 0 on success or -ENOENT, if there's no more seg descriptors in + * ec_cmd, or other negative error code. For other error codes cmd status + * and sense supposed to be set. + */ +static int scst_cm_setup_seg_descr(struct scst_cmd *ec_cmd) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + struct scst_ext_copy_seg_descr *sd; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p, cm_cur_seg_descr %d", ec_cmd, + priv->cm_cur_seg_descr); + + EXTRACHECKS_BUG_ON(priv->cm_cur_seg_descr > ec_cmd->cmd_data_descriptors_cnt); + + while (1) { + if (priv->cm_cur_seg_descr == ec_cmd->cmd_data_descriptors_cnt) + goto out_enoent; + + sd = &priv->cm_seg_descrs[priv->cm_cur_seg_descr]; + EXTRACHECKS_BUG_ON(sd->type != SCST_EXT_COPY_SEG_DATA); + if (sd->data_descr.data_len != 0) + break; + + priv->cm_cur_seg_descr++; + TRACE_DBG("ec_cmd %p, cm_cur_seg_descr %d", ec_cmd, + priv->cm_cur_seg_descr); + } + + if (priv->cm_list_id != NULL) { + /* SCSI: including the being processed one */ + priv->cm_list_id->cm_segs_processed = priv->cm_cur_seg_descr + 1; + } + + res = 0; + +out: + TRACE_EXIT_RES(res); + return res; + +out_enoent: + TRACE_DBG("ec_cmd %p finished", ec_cmd); + res = -ENOENT; + goto out; +} + +/* cm_mutex suppose to be locked or no activities on this ec_cmd's priv */ +static void scst_cm_advance_seg_descr(struct scst_cmd *ec_cmd) +{ + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p", ec_cmd); + + EXTRACHECKS_BUG_ON(priv->cm_cur_in_flight != 0); + + scst_cm_destroy_data_descrs(ec_cmd); + + priv->cm_cur_seg_descr++; + + TRACE_EXIT(); + return; +} + +static void scst_cm_prepare_final_sense(struct scst_cmd *ec_cmd) +{ + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + uint8_t *fsense = NULL; + int d_sense = scst_get_cmd_dev_d_sense(ec_cmd); + bool copy_sense = false; + int sense_to_copy = ec_cmd->sense_valid_len; + int sense_len = ec_cmd->sense_valid_len; + + TRACE_ENTRY(); + + if (likely(priv->cm_error == SCST_CM_ERROR_NONE)) + goto out; + + TRACE_DBG("ec_cmd %p, cm_error %d, sense_to_copy %d", ec_cmd, + priv->cm_error, sense_to_copy); + + if (sense_to_copy > (SCST_SENSE_BUFFERSIZE - 18)) { + PRINT_WARNING("Too small sense buffer, %d bytes will be " + "truncated (ec_cmd %p)", + sense_to_copy - (SCST_SENSE_BUFFERSIZE-18), ec_cmd); + sense_to_copy = SCST_SENSE_BUFFERSIZE - 18; + } + + if ((priv->cm_error == SCST_CM_ERROR_WRITE) && + (ec_cmd->status != SAM_STAT_CHECK_CONDITION)) { + int rc; + struct scst_ext_copy_seg_descr *sd = &priv->cm_seg_descrs[priv->cm_cur_seg_descr]; + + /* THIRD PARTY DEVICE FAILURE */ + + rc = scst_alloc_sense(ec_cmd, 0); + if (rc != 0) + goto out; + + TRACE_DBG("d_sense %d, cm_cur_seg_descr %d, cur_data_descr %d, " + "tgt_descr_offs %d", d_sense, priv->cm_cur_seg_descr, + priv->cm_cur_data_descr, sd->tgt_descr_offs); + + if (d_sense) { + /* Descriptor format */ + ec_cmd->sense[0] = 0x72; + ec_cmd->sense[1] = COPY_ABORTED; + ec_cmd->sense[2] = 0xD; /* ASC */ + ec_cmd->sense[3] = 1; /* ASCQ */ + ec_cmd->sense[7] = 20; /* additional Sense Length */ + + ec_cmd->sense[8] = 1; /* Command specific descriptor */ + ec_cmd->sense[9] = 0xA; + put_unaligned_be16(priv->cm_cur_seg_descr, &ec_cmd->sense[14]); + + ec_cmd->sense[20] = 2; /* Sense key specific descriptor */ + ec_cmd->sense[21] = 6; + ec_cmd->sense[24] = 0x80; + put_unaligned_be16(sd->tgt_descr_offs, &ec_cmd->sense[25]); + + ec_cmd->sense_valid_len = 16; + } else { + /* Fixed format */ + ec_cmd->sense[0] = 0x70; + ec_cmd->sense[2] = COPY_ABORTED; + ec_cmd->sense[7] = 0x0a; /* additional Sense Length */ + ec_cmd->sense[12] = 0xD; /* ASC */ + ec_cmd->sense[13] = 1; /* ASCQ */ + + put_unaligned_be16(priv->cm_cur_seg_descr, &ec_cmd->sense[10]); + + ec_cmd->sense[15] = 0x80; + put_unaligned_be16(sd->tgt_descr_offs, &ec_cmd->sense[16]); + + ec_cmd->sense_valid_len = 18; + } + + ec_cmd->status = SAM_STAT_CHECK_CONDITION; + goto out; + } + + fsense = mempool_alloc(scst_sense_mempool, GFP_KERNEL); + if (fsense == NULL) { + PRINT_ERROR("Allocation of the intermediate Extended Copy " + "sense buffer failed. Reported sense data can be " + "incorrect (ec_cmd %p)", ec_cmd); + goto out; + } + memset(fsense, 0, SCST_SENSE_BUFFERSIZE); + + if (d_sense) { + /* Descriptor format */ + fsense[0] = 0x72; + fsense[1] = COPY_ABORTED; + fsense[7] = 12; /* additional Sense Length */ + + fsense[8] = 1; /* Command specific descriptor */ + fsense[9] = 0xA; + put_unaligned_be16(priv->cm_cur_seg_descr, &fsense[14]); + + sense_len = 20; + } else { + /* Fixed format */ + fsense[0] = 0x70; + fsense[2] = COPY_ABORTED; + fsense[7] = 0x0a; /* additional Sense Length */ + + put_unaligned_be16(priv->cm_cur_seg_descr, &fsense[10]); + + if (priv->cm_error == SCST_CM_ERROR_READ) { + fsense[8] = 18; + copy_sense = scst_sense_valid(ec_cmd->sense); + } else if (priv->cm_error == SCST_CM_ERROR_WRITE) { + fsense[9] = 18; + copy_sense = scst_sense_valid(ec_cmd->sense); + } else + sBUG(); + + if (copy_sense) { + TRACE_DBG("Copying %db of old sense", sense_to_copy); + fsense[7] += 1 + sense_to_copy; + fsense[17] = ec_cmd->status; + memcpy(&fsense[18], ec_cmd->sense, sense_to_copy); + } + + sense_len = fsense[7] + 8; + TRACE_DBG("New sense len %d", sense_len); + } + + ec_cmd->status = SAM_STAT_CHECK_CONDITION; + if (ec_cmd->sense != NULL) { + memcpy(ec_cmd->sense, fsense, sense_len); + ec_cmd->sense_valid_len = sense_len; + } else + scst_alloc_set_sense(ec_cmd, 0, fsense, sense_len); + + mempool_free(fsense, scst_sense_mempool); + +out: + TRACE_EXIT(); + return; +} + +static void scst_cm_store_list_id_details(struct scst_cmd *ec_cmd) +{ + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + struct scst_cm_list_id *l = priv->cm_list_id; + + TRACE_ENTRY(); + + if (l != NULL) { + TRACE_DBG("List id %p done (status %d, sense valid %d, sense " + "len %d)", l, ec_cmd->status, scst_sense_valid(ec_cmd->sense), + ec_cmd->sense_valid_len); + spin_lock_irq(&scst_cm_lock); + l->cm_list_id_state = SCST_CM_LIST_ID_STATE_DONE; + if (ec_cmd->status != 0) { + l->cm_status = ec_cmd->status; + if (scst_sense_valid(ec_cmd->sense)) { + int len = ec_cmd->sense_valid_len; + if (len > sizeof(l->cm_sense)) { + PRINT_WARNING("EC command's sense is " + "too big (%d) with max allowed " + "%d, truncating", len, + (int)sizeof(l->cm_sense)); + len = sizeof(l->cm_sense); + } + l->cm_sense_len = ec_cmd->sense_valid_len; + memcpy(l->cm_sense, ec_cmd->sense, len); + } + } + spin_unlock_irq(&scst_cm_lock); + } + + TRACE_EXIT(); + return; +} + +static void scst_cm_ec_cmd_done(struct scst_cmd *ec_cmd) +{ +#ifdef CONFIG_SCST_EXTRACHECKS + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; +#endif + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p finished with status %d", ec_cmd, ec_cmd->status); + + EXTRACHECKS_BUG_ON(priv->cm_cur_in_flight != 0); + EXTRACHECKS_BUG_ON(priv->cm_data_descrs != NULL); + + scst_cm_prepare_final_sense(ec_cmd); + scst_cm_store_list_id_details(ec_cmd); + + ec_cmd->completed = 1; /* for success */ + ec_cmd->scst_cmd_done(ec_cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD); + + TRACE_EXIT(); + return; +} + +static void scst_cm_ec_sched_next_seg(struct scst_cmd *ec_cmd) +{ + TRACE_ENTRY(); + + /* No lock is needed, because it's all supposed to be calm */ + + scst_cm_advance_seg_descr(ec_cmd); + + /* + * There's recursion possible here, if dev handlers call after + * remapping scst_ext_copy_remap_done() from the same thread context. + */ + scst_cm_ext_copy_exec(ec_cmd); + + TRACE_EXIT(); + return; +} + +static void scst_cm_in_flight_cmd_finished(struct scst_cmd *ec_cmd) +{ + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + int f; + + TRACE_ENTRY(); + + mutex_lock(&priv->cm_mutex); + + priv->cm_cur_in_flight--; + f = priv->cm_cur_in_flight; + + mutex_unlock(&priv->cm_mutex); + + TRACE_DBG("ec_cmd %p, priv->cm_cur_in_flight %d", ec_cmd, f); + + if (f > 0) + goto out; + + if (priv->cm_list_id != NULL) + priv->cm_list_id->cm_written_size += priv->cm_written; + + scst_cm_ec_sched_next_seg(ec_cmd); + +out: + TRACE_EXIT(); + return; +} + +static int scst_cm_add_to_internal_cmd_list(struct scst_cmd *cmd, + struct scst_cmd *ec_cmd, struct scst_cmd *orig_cmd, + scst_i_finish_fn_t finish_fn) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + struct scst_cm_internal_cmd_priv *p; + + TRACE_ENTRY(); + + EXTRACHECKS_BUG_ON(ec_cmd == cmd); + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (p == NULL) { + PRINT_ERROR("Unable to alloc scst_cm_internal_cmd_priv " + "(size %d)", (int)sizeof(*p)); + goto out_enomem; + } + + p->cm_finish_fn = finish_fn; + p->cm_orig_cmd = orig_cmd; + p->cm_cmd = cmd; + + TRACE_DBG("Adding internal cmd %p (priv %p, ec_cmd %p, orig_cmd %p)", + cmd, p, ec_cmd, orig_cmd); + spin_lock_irq(&scst_cm_lock); + list_add_tail(&p->cm_internal_cmd_list_entry, &priv->cm_internal_cmd_list); + spin_unlock_irq(&scst_cm_lock); + + cmd->tgt_i_priv = p; + + res = 0; + +out: + TRACE_EXIT_RES(res); + return res; + +out_enomem: + scst_set_busy(ec_cmd); + res = -ENOMEM; + goto out; +} + +static void scst_cm_del_free_from_internal_cmd_list(struct scst_cmd *cmd, + bool unblock_dev) +{ + struct scst_cm_internal_cmd_priv *p = cmd->tgt_i_priv; + + TRACE_ENTRY(); + + TRACE_DBG("Deleting/freeing internal cmd %p (op %s, priv %p, " + "orig_cmd %p)", cmd, scst_get_opcode_name(cmd), p, + p->cm_orig_cmd); + + spin_lock_irq(&scst_cm_lock); + list_del(&p->cm_internal_cmd_list_entry); + spin_unlock_irq(&scst_cm_lock); + + if (unblock_dev) { + TRACE_DBG("dev %p (internal cmd %p)", cmd->dev, cmd); + scst_check_unblock_dev(cmd); + } + + cmd->tgt_i_priv = NULL; + + kfree(p); + + TRACE_EXIT(); + return; +} + +static void scst_cm_read_cmd_finished(struct scst_cmd *rcmd); + +/* cm_mutex suppose to be locked */ +static int __scst_cm_push_single_read(struct scst_cmd *ec_cmd, + int64_t lba, int blocks) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + uint8_t read_cdb[32]; + struct scst_device *rdev = priv->cm_read_tgt_dev->dev; + int block_shift = rdev->block_shift; + int len = blocks << block_shift; + struct scst_cmd *rcmd; + int cdb_len; + bool check_dif = (rdev->dev_dif_mode & SCST_DIF_MODE_DEV); + + TRACE_ENTRY(); + + if (unlikely(scst_cm_is_ec_cmd_done(ec_cmd))) { + TRACE_MGMT_DBG("EC cmd %p done: aborting further read " + "commands", ec_cmd); + priv->cm_left_to_read = 0; + res = -EPIPE; + goto out; + } + + memset(read_cdb, 0, sizeof(read_cdb)); + if (rdev->dev_dif_type != 2 || !check_dif) { + read_cdb[0] = READ_16; + if (check_dif) + read_cdb[1] |= 0x20; + put_unaligned_be64(lba, &read_cdb[2]); + put_unaligned_be32(blocks, &read_cdb[10]); + cdb_len = 16; + } else { + read_cdb[0] = VARIABLE_LENGTH_CMD; + put_unaligned_be16(SUBCODE_READ_32, &read_cdb[8]); + read_cdb[7] = 0x18; + cdb_len = 32; + read_cdb[10] = 0x20; + put_unaligned_be64(lba, &read_cdb[12]); + put_unaligned_be32(blocks, &read_cdb[28]); + put_unaligned_be32(lba & 0xFFFF, &read_cdb[20]); + /* No app tag check */ + } + + rcmd = __scst_create_prepare_internal_cmd(read_cdb, + cdb_len, SCST_CMD_QUEUE_SIMPLE, + priv->cm_read_tgt_dev, GFP_KERNEL, false); + if (rcmd == NULL) { + res = -ENOMEM; + goto out_busy; + } + + rcmd->internal_check_local_events = 1; + + rcmd->expected_data_direction = SCST_DATA_READ; + rcmd->expected_transfer_len_full = len; + if (check_dif != 0) + rcmd->expected_transfer_len_full += len >> (block_shift - SCST_DIF_TAG_SHIFT); + rcmd->expected_values_set = 1; + + res = scst_cm_add_to_internal_cmd_list(rcmd, ec_cmd, ec_cmd, + scst_cm_read_cmd_finished); + if (res != 0) + goto out_free_rcmd; + + TRACE_DBG("Adding ec_cmd's (%p) READ rcmd %p (lba %lld, blocks %d, " + "check_dif %d) to active cmd list", ec_cmd, rcmd, + (long long)rcmd->lba, blocks, check_dif); + spin_lock_irq(&rcmd->cmd_threads->cmd_list_lock); + list_add_tail(&rcmd->cmd_list_entry, &rcmd->cmd_threads->active_cmd_list); + spin_unlock_irq(&rcmd->cmd_threads->cmd_list_lock); + + res = 0; + +out: + TRACE_EXIT_RES(res); + return res; + +out_busy: + scst_set_busy(ec_cmd); + goto out; + +out_free_rcmd: + __scst_cmd_put(rcmd); + goto out; +} + +static void scst_cm_read_retry_fn(struct scst_cmd *rcmd) +{ + struct scst_cm_internal_cmd_priv *p = rcmd->tgt_i_priv; + struct scst_cmd *ec_cmd = p->cm_orig_cmd; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + int rc; + + TRACE_ENTRY(); + + mutex_lock(&priv->cm_mutex); + + rc = __scst_cm_push_single_read(ec_cmd, rcmd->lba, + rcmd->data_len >> priv->cm_read_tgt_dev->dev->block_shift); + + /* ec_cmd can get dead after we will drop cm_mutex! */ + scst_cm_del_free_from_internal_cmd_list(rcmd, false); + + mutex_unlock(&priv->cm_mutex); + + if (rc == 0) + wake_up(&priv->cm_read_tgt_dev->active_cmd_threads->cmd_list_waitQ); + else + scst_cm_in_flight_cmd_finished(ec_cmd); + + TRACE_EXIT(); + return; +} + +static int scst_cm_push_single_write(struct scst_cmd *ec_cmd, + int64_t lba, int blocks, struct scst_cmd *rcmd); + +static void scst_cm_write_retry_fn(struct scst_cmd *wcmd) +{ + struct scst_cm_internal_cmd_priv *p = wcmd->tgt_i_priv; + struct scst_cmd *rcmd = p->cm_orig_cmd; + struct scst_cm_internal_cmd_priv *rp = rcmd->tgt_i_priv; + struct scst_cmd *ec_cmd = rp->cm_orig_cmd; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + int rc; + + TRACE_ENTRY(); + + mutex_lock(&priv->cm_mutex); + + rc = scst_cm_push_single_write(ec_cmd, wcmd->lba, + wcmd->data_len >> priv->cm_write_tgt_dev->dev->block_shift, + rcmd); + + /* ec_cmd can get dead after we will drop cm_mutex! */ + scst_cm_del_free_from_internal_cmd_list(wcmd, false); + + mutex_unlock(&priv->cm_mutex); + + __scst_cmd_put(rcmd); + + if (rc != 0) + scst_cm_in_flight_cmd_finished(ec_cmd); + + TRACE_EXIT(); + return; +} + +static int scst_cm_push_single_read(struct scst_cmd *ec_cmd, int blocks, + bool inc_cur_in_flight); + +static void scst_cm_write_cmd_finished(struct scst_cmd *wcmd) +{ + struct scst_cm_internal_cmd_priv *p = wcmd->tgt_i_priv; + struct scst_cmd *rcmd = p->cm_orig_cmd; + struct scst_cm_internal_cmd_priv *rp = rcmd->tgt_i_priv; + struct scst_cmd *ec_cmd = rp->cm_orig_cmd; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + int rc, blocks; + + TRACE_ENTRY(); + + TRACE_DBG("Write cmd %p finished (ec_cmd %p, rcmd %p, cm_cur_in_flight %d)", + wcmd, rcmd, ec_cmd, priv->cm_cur_in_flight); + + EXTRACHECKS_BUG_ON(wcmd->cdb[0] != WRITE_16); + + if (unlikely(scst_cm_is_ec_cmd_done(ec_cmd))) + goto out_finished; + + rc = scst_cm_err_check_retry(wcmd, ec_cmd->start_time, scst_cm_write_retry_fn); + if (likely(rc == SCST_CM_STATUS_CMD_SUCCEEDED)) + goto cont; + else if (rc == SCST_CM_STATUS_RETRY) + goto out; + else { + TRACE_DBG("Write cmd %p (ec_cmd %p) finished not successfully", + wcmd, ec_cmd); + if (wcmd->status == SAM_STAT_CHECK_CONDITION) + rc = scst_set_cmd_error_sense(ec_cmd, wcmd->sense, + wcmd->sense_valid_len); + else { + sBUG_ON(wcmd->sense != NULL); + rc = scst_set_cmd_error_status(ec_cmd, wcmd->status); + } + if (rc != 0) { + /* + * UAs should be retried unconditionally, but during + * limited time. If we are here, we need to requeue it. + */ + WARN_ON(scst_is_ua_sense(wcmd->sense, wcmd->sense_valid_len)); + sBUG_ON(priv->cm_error == SCST_CM_ERROR_NONE); + } else + priv->cm_error = SCST_CM_ERROR_WRITE; + goto out_finished; + } + +cont: + priv->cm_written += wcmd->data_len; + TRACE_DBG("ec_cmd %p, cm_written %lld (data_len %lld)", ec_cmd, + (long long)priv->cm_written, (long long)wcmd->data_len); + + wcmd->sg = NULL; + wcmd->sg_cnt = 0; + + mutex_lock(&priv->cm_mutex); + + if (priv->cm_left_to_read == 0) { + if (priv->cm_cur_data_descr >= priv->cm_data_descrs_cnt) + goto out_unlock_finished; + + rc = scst_cm_setup_next_data_descr(ec_cmd); + if (rc != 0) + goto out_unlock_finished; + } + + EXTRACHECKS_BUG_ON(priv->cm_left_to_read == 0); + + blocks = min_t(int, priv->cm_left_to_read, priv->cm_max_each_read); + + rc = scst_cm_push_single_read(ec_cmd, blocks, false); + if (rc != 0) + goto out_unlock_finished; + + scst_cm_del_free_from_internal_cmd_list(wcmd, false); + scst_cm_del_free_from_internal_cmd_list(rcmd, false); + + mutex_unlock(&priv->cm_mutex); + + wake_up(&rcmd->cmd_threads->cmd_list_waitQ); + +out_put: + __scst_cmd_put(rcmd); + +out: + TRACE_EXIT(); + return; + +out_unlock_finished: + mutex_unlock(&priv->cm_mutex); + +out_finished: + scst_cm_del_free_from_internal_cmd_list(wcmd, false); + scst_cm_del_free_from_internal_cmd_list(rcmd, false); + + scst_cm_in_flight_cmd_finished(ec_cmd); + goto out_put; +} + +static int scst_cm_push_single_write(struct scst_cmd *ec_cmd, + int64_t lba, int blocks, struct scst_cmd *rcmd) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + uint8_t write16_cdb[16]; + struct scst_cmd *wcmd; + int len; + + TRACE_ENTRY(); + + len = blocks << priv->cm_write_tgt_dev->dev->block_shift; + + /* + * ToDo: if rcmd is coming with tags SG, use it after updating ref and + * app tags instead of regenerating guard tags again with WRITE(16) + */ + + memset(write16_cdb, 0, sizeof(write16_cdb)); + write16_cdb[0] = WRITE_16; + put_unaligned_be64(lba, &write16_cdb[2]); + put_unaligned_be32(blocks, &write16_cdb[10]); + + wcmd = __scst_create_prepare_internal_cmd(write16_cdb, + sizeof(write16_cdb), SCST_CMD_QUEUE_SIMPLE, + priv->cm_write_tgt_dev, GFP_KERNEL, false); + if (wcmd == NULL) { + res = -ENOMEM; + goto out_busy; + } + + wcmd->internal_check_local_events = 1; + + wcmd->expected_data_direction = SCST_DATA_WRITE; + wcmd->expected_transfer_len_full = len; + wcmd->expected_values_set = 1; + + res = scst_cm_add_to_internal_cmd_list(wcmd, ec_cmd, rcmd, + scst_cm_write_cmd_finished); + if (res != 0) + goto out_free_wcmd; + + __scst_cmd_get(rcmd); + + wcmd->tgt_i_sg = rcmd->sg; + wcmd->tgt_i_sg_cnt = rcmd->sg_cnt; + wcmd->tgt_i_data_buf_alloced = 1; + + TRACE_DBG("Adding EC (%p) WRITE(16) cmd %p (lba %lld, blocks %d) to " + "active cmd list", ec_cmd, wcmd, (long long)wcmd->lba, blocks); + spin_lock_irq(&wcmd->cmd_threads->cmd_list_lock); + list_add_tail(&wcmd->cmd_list_entry, &wcmd->cmd_threads->active_cmd_list); + wake_up(&wcmd->cmd_threads->cmd_list_waitQ); + spin_unlock_irq(&wcmd->cmd_threads->cmd_list_lock); + + res = 0; + +out: + TRACE_EXIT_RES(res); + return res; + +out_busy: + scst_set_busy(ec_cmd); + goto out; + +out_free_wcmd: + __scst_cmd_put(wcmd); + goto out; +} + +static void scst_cm_read_cmd_finished(struct scst_cmd *rcmd) +{ + struct scst_cm_internal_cmd_priv *p = rcmd->tgt_i_priv; + struct scst_cmd *ec_cmd = p->cm_orig_cmd; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + int64_t lba; + int rc, len, blocks; + + TRACE_ENTRY(); + + TRACE_DBG("READ cmd %p finished (ec_cmd %p, p %p)", rcmd, ec_cmd, p); + + if (unlikely(scst_cm_is_ec_cmd_done(ec_cmd))) + goto out_finished; + + rc = scst_cm_err_check_retry(rcmd, ec_cmd->start_time, scst_cm_read_retry_fn); + if (likely(rc == SCST_CM_STATUS_CMD_SUCCEEDED)) + goto cont; + else if (rc == SCST_CM_STATUS_RETRY) + goto out; + else { + TRACE_DBG("Read cmd %p (ec_cmd %p) finished not successfully", + rcmd, ec_cmd); + if (rcmd->status == SAM_STAT_CHECK_CONDITION) + rc = scst_set_cmd_error_sense(ec_cmd, rcmd->sense, + rcmd->sense_valid_len); + else { + sBUG_ON(rcmd->sense != NULL); + rc = scst_set_cmd_error_status(ec_cmd, rcmd->status); + } + if (rc != 0) { + /* + * UAs should be retried unconditionally, but during + * limited time. If we are here, we need to requeue it. + */ + WARN_ON(scst_is_ua_sense(rcmd->sense, rcmd->sense_valid_len)); + sBUG_ON(priv->cm_error == SCST_CM_ERROR_NONE); + } else + priv->cm_error = SCST_CM_ERROR_READ; + goto out_finished; + } + +cont: + lba = rcmd->lba - priv->cm_start_read_lba; + lba <<= priv->cm_read_tgt_dev->dev->block_shift; + lba >>= priv->cm_write_tgt_dev->dev->block_shift; + lba += priv->cm_start_write_lba; + + len = rcmd->data_len; + blocks = len >> priv->cm_write_tgt_dev->dev->block_shift; + + TRACE_DBG("rcmd->lba %lld, start_read_lba %lld, read shift %d, write " + "shift %d, start_write_lba %lld, lba %lld, len %d, blocks %d", + (long long)rcmd->lba, (long long)priv->cm_start_read_lba, + priv->cm_read_tgt_dev->dev->block_shift, + priv->cm_write_tgt_dev->dev->block_shift, + (long long)priv->cm_start_write_lba, lba, len, blocks); + + rc = scst_cm_push_single_write(ec_cmd, lba, blocks, rcmd); + if (rc != 0) + goto out_finished; + +out: + TRACE_EXIT(); + return; + +out_finished: + scst_cm_del_free_from_internal_cmd_list(rcmd, false); + + scst_cm_in_flight_cmd_finished(ec_cmd); + goto out; +} + +/* cm_mutex suppose to be locked */ +static int scst_cm_push_single_read(struct scst_cmd *ec_cmd, int blocks, + bool inc_cur_in_flight) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p, cm_cur_read_lba %lld, cm_left_to_read %d, " + "blocks %d", ec_cmd, (long long)priv->cm_cur_read_lba, + priv->cm_left_to_read, blocks); + + res = __scst_cm_push_single_read(ec_cmd, priv->cm_cur_read_lba, blocks); + if (res != 0) + goto out; + + priv->cm_cur_read_lba += blocks; + priv->cm_left_to_read -= blocks; + + if (inc_cur_in_flight) { + priv->cm_cur_in_flight++; + TRACE_DBG("ec_cmd %p, new cm_cur_in_flight %d", ec_cmd, + priv->cm_cur_in_flight); + } + +out: + TRACE_EXIT_RES(res); + return res; +} + +/* + * Generates original bunch of internal READ commands. In case of error + * directly finishes ec_cmd, so it might be dead upon return! + */ +static void scst_cm_gen_reads(struct scst_cmd *ec_cmd) +{ + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + int cnt = 0; + + TRACE_ENTRY(); + + mutex_lock(&priv->cm_mutex); + + while (1) { + int rc; + while ((priv->cm_left_to_read > 0) && + (priv->cm_cur_in_flight < SCST_MAX_IN_FLIGHT_INTERNAL_COMMANDS)) { + int blocks; + + blocks = min_t(int, priv->cm_left_to_read, priv->cm_max_each_read); + + rc = scst_cm_push_single_read(ec_cmd, blocks, true); + if (rc != 0) + goto out_err; + + cnt++; + } + + if (priv->cm_cur_in_flight == SCST_MAX_IN_FLIGHT_INTERNAL_COMMANDS) + break; + + rc = scst_cm_setup_next_data_descr(ec_cmd); + if (rc != 0) + goto out_err; + } + + EXTRACHECKS_BUG_ON(cnt == 0); + +out_wake: + if (cnt != 0) + wake_up(&priv->cm_read_tgt_dev->active_cmd_threads->cmd_list_waitQ); + + mutex_unlock(&priv->cm_mutex); + +out: + TRACE_EXIT(); + return; + +out_err: + if (priv->cm_cur_in_flight != 0) + goto out_wake; + else { + mutex_unlock(&priv->cm_mutex); + scst_cm_ec_cmd_done(ec_cmd); + } + goto out; +} + +/* cm_mutex suppose to be locked or no activities on this ec_cmd's priv */ +static void scst_cm_process_data_descrs(struct scst_cmd *ec_cmd, + const struct scst_ext_copy_data_descr *dds, int dds_cnt) +{ + int rc; + + TRACE_ENTRY(); + + rc = scst_cm_setup_data_descrs(ec_cmd, dds, dds_cnt); + if (rc != 0) + goto out_done; + + scst_cm_gen_reads(ec_cmd); + + /* ec_cmd can be dead here! */ + +out: + TRACE_EXIT(); + return; + +out_done: + scst_cm_ec_cmd_done(ec_cmd); + goto out; +} + +/** + * scst_ext_copy_get_cur_seg_data_len() - return current segment data len + * @ec_cmd: EXTENDED COPY command + */ +int scst_ext_copy_get_cur_seg_data_len(struct scst_cmd *ec_cmd) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + res = priv->cm_seg_descrs[priv->cm_cur_seg_descr].data_descr.data_len; + + TRACE_EXIT_RES(res); + return res; +} +EXPORT_SYMBOL_GPL(scst_ext_copy_get_cur_seg_data_len); + +/** + * scst_ext_copy_remap_done() - dev handler done with remapping this segment + * @ec_cmd: EXTENDED COPY command + * @dds: Leftover data descriptors + * @dds_cnt: Count of the leftover data descriptors + * + * Called by dev handlers from inside ext_copy_remap() callback upon finish. + * All not finished spaces should be k?alloc() as array of data descriptors + * in dds argument with count dds_count. SCST core then will copy them using + * internal copy machine and then kfree() dds. + * + * dds can point to &descr->data_descr, where descr is pointer supplied to + * ext_copy_remap(). In this case SCST core will not kfree() it. + * + * If dds is NULL, then all data have been remapped, so SCST core will switch + * to the next segment descriptor, if any. + */ +void scst_ext_copy_remap_done(struct scst_cmd *ec_cmd, + struct scst_ext_copy_data_descr *dds, int dds_cnt) +{ + TRACE_ENTRY(); + + scst_set_exec_time(ec_cmd); + + if (dds == NULL) + scst_cm_ec_sched_next_seg(ec_cmd); + else + scst_cm_process_data_descrs(ec_cmd, dds, dds_cnt); + + /* ec_cmd can be dead here! */ + + TRACE_EXIT(); + return; +} +EXPORT_SYMBOL_GPL(scst_ext_copy_remap_done); + +static int scst_cm_try_to_remap(struct scst_cmd *ec_cmd); + +static void scst_cm_remap_retry_fn(struct scst_cmd *cmd) +{ + struct scst_cmd *ec_cmd = cmd->tgt_i_priv; + int rc; + + TRACE_ENTRY(); + + rc = scst_cm_try_to_remap(ec_cmd); + sBUG_ON(rc != 0); + + TRACE_EXIT(); + return; +} + +/* + * Tries to remap data before copying. + * + * Returns !=0 if remapping is not possible, or not 0 otherwise. + */ +static int scst_cm_try_to_remap(struct scst_cmd *ec_cmd) +{ + int res, rc; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + struct scst_ext_copy_seg_descr *sd = &priv->cm_seg_descrs[priv->cm_cur_seg_descr]; + struct scst_ext_copy_data_descr *dd = &sd->data_descr; + struct scst_dev_type *handler = ec_cmd->dev->handler; + uint8_t cdb[16]; + struct scst_cmd *cmd; + + TRACE_ENTRY(); + + if (handler->ext_copy_remap == NULL) { + res = 1; + goto out; + } + + res = 0; + + /* !! priv data descriptors fields are not setup yet !! */ + + TRACE_DBG("Checking reservations on read dev %s (ec_cmd %p)", + sd->src_tgt_dev->dev->virt_name, ec_cmd); + + memset(cdb, 0, sizeof(cdb)); + cdb[0] = READ_16; + put_unaligned_be64(dd->src_lba, &cdb[2]); + put_unaligned_be32(dd->data_len >> sd->src_tgt_dev->dev->block_shift, &cdb[10]); + + cmd = __scst_create_prepare_internal_cmd(cdb, + sizeof(cdb), SCST_CMD_QUEUE_SIMPLE, + sd->src_tgt_dev, GFP_KERNEL, true); + if (cmd == NULL) + goto out_busy; + + cmd->internal_check_local_events = 1; + cmd->tgt_i_priv = ec_cmd; /* needed for retries */ + + rc = __scst_check_local_events(cmd, false); + if (unlikely(rc != 0)) + goto out_check_retry; + + __scst_cmd_put(cmd); + + TRACE_DBG("Checking reservations on write dev %s (ec_cmd %p)", + sd->dst_tgt_dev->dev->virt_name, ec_cmd); + + memset(cdb, 0, sizeof(cdb)); + cdb[0] = WRITE_16; + put_unaligned_be64(dd->dst_lba, &cdb[2]); + put_unaligned_be32(dd->data_len >> sd->dst_tgt_dev->dev->block_shift, &cdb[10]); + + cmd = __scst_create_prepare_internal_cmd(cdb, + sizeof(cdb), SCST_CMD_QUEUE_SIMPLE, + sd->dst_tgt_dev, GFP_KERNEL, true); + if (cmd == NULL) + goto out_busy; + + cmd->internal_check_local_events = 1; + cmd->tgt_i_priv = ec_cmd; /* needed for retries */ + + rc = __scst_check_local_events(cmd, false); + if (unlikely(rc != 0)) + goto out_check_retry; + + __scst_cmd_put(cmd); + + TRACE_DBG("Calling ext_copy_remap() for dev %s (ec_cmd %p)", + sd->dst_tgt_dev->dev->virt_name, ec_cmd); + + scst_set_exec_start(ec_cmd); + handler->ext_copy_remap(ec_cmd, sd); + +out: + TRACE_EXIT_RES(res); + return res; + +out_check_retry: + rc = scst_cm_err_check_retry(cmd, ec_cmd->start_time, scst_cm_remap_retry_fn); + sBUG_ON(rc == SCST_CM_STATUS_CMD_SUCCEEDED); + if (rc == SCST_CM_STATUS_CMD_FAILED) { + TRACE_DBG("Remap check cmd %p (ec_cmd %p, op %s) failed", + cmd, ec_cmd, scst_get_opcode_name(cmd)); + if (cmd->status == SAM_STAT_CHECK_CONDITION) + rc = scst_set_cmd_error_sense(ec_cmd, cmd->sense, + cmd->sense_valid_len); + else { + sBUG_ON(cmd->sense != NULL); + rc = scst_set_cmd_error_status(ec_cmd, cmd->status); + } + if (rc != 0) { + /* + * UAs should be retried unconditionally, but during + * limited time. If we are here, we need to requeue it. + */ + WARN_ON(scst_is_ua_sense(cmd->sense, cmd->sense_valid_len)); + } else { + if (cmd->cdb[0] == READ_16) + priv->cm_error = SCST_CM_ERROR_READ; + else { + EXTRACHECKS_BUG_ON(cmd->cdb[0] != WRITE_16); + priv->cm_error = SCST_CM_ERROR_WRITE; + } + } + __scst_cmd_put(cmd); + goto out_done; + } + __scst_cmd_put(cmd); + goto out; + +out_busy: + res = -ENOMEM; + scst_set_busy(ec_cmd); + +out_done: + scst_cm_ec_cmd_done(ec_cmd); + goto out; +} + +static void scst_cm_process_cur_seg_descr(struct scst_cmd *ec_cmd) +{ + int rc; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + rc = scst_cm_try_to_remap(ec_cmd); + if (rc == 0) + goto out; + + /* No remapping supported */ + + scst_ext_copy_remap_done(ec_cmd, + &priv->cm_seg_descrs[priv->cm_cur_seg_descr].data_descr, 1); + +out: + TRACE_EXIT(); + return; +} + +int scst_cm_ext_copy_exec(struct scst_cmd *ec_cmd) +{ + int res = SCST_EXEC_COMPLETED, rc; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + if (unlikely(priv == NULL)) + goto out_local_done; + + if (unlikely(scst_cm_is_ec_cmd_done(ec_cmd))) { + TRACE_DBG("ec_cmd %p done", ec_cmd); + goto out_done; + } + + rc = scst_cm_setup_seg_descr(ec_cmd); + if (rc != 0) + goto out_err_done; + + scst_cm_process_cur_seg_descr(ec_cmd); + +out: + TRACE_EXIT(); + return res; + +out_local_done: + ec_cmd->completed = 1; /* for success */ + ec_cmd->scst_cmd_done(ec_cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD); + goto out; + +out_err_done: + sBUG_ON((rc != -ENOENT) && !ec_cmd->completed); + +out_done: + scst_cm_ec_cmd_done(ec_cmd); + goto out; +} + +bool scst_cm_ec_cmd_overlap(struct scst_cmd *ec_cmd, struct scst_cmd *cmd) +{ + bool res = false; + int i; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + TRACE_DBG("ec_cmd %p, cmd %p", ec_cmd, cmd); + + EXTRACHECKS_BUG_ON(ec_cmd->cdb[0] != EXTENDED_COPY); + + if ((cmd->op_flags & SCST_LBA_NOT_VALID) != 0) + goto out; + + for (i = 0; i < ec_cmd->cmd_data_descriptors_cnt; i++) { + struct scst_ext_copy_seg_descr *sd = &priv->cm_seg_descrs[i]; + + TRACE_DBG("type %d, dst_dev %p, dev %p", sd->type, + sd->dst_tgt_dev->dev, cmd->dev); + + if (sd->type != SCST_EXT_COPY_SEG_DATA) + continue; + if (sd->dst_tgt_dev->dev != cmd->dev) + continue; + + res = scst_lba1_inside_lba2(sd->data_descr.dst_lba, cmd->lba, + cmd->data_len >> cmd->dev->block_shift); + if (res) + goto out; + res = scst_lba1_inside_lba2(cmd->lba, sd->data_descr.dst_lba, + sd->data_descr.data_len >> sd->dst_tgt_dev->dev->block_shift); + if (res) + goto out; + } + +out: + TRACE_EXIT_RES(res); + return res; +} + +/* + * No locks. Returns true if cmd should be blocked. + */ +bool scst_cm_check_block_all_devs(struct scst_cmd *cmd) +{ + bool res = false; + struct scst_cmd *ec_cmd; + struct scst_cm_ec_cmd_priv *d; + struct scst_cm_dev_entry *e; + + TRACE_ENTRY(); + + /* + * EXTENDED COPY command creates a new challenge: it can operate on + * several devices at the same time. Hence, it opens a wide possibility + * of deadlocks like the following. + * + * Device A received EXTENDED COPY command EC1 to copy data from device + * A to device B. It started doing the job by generating SCSI READ and + * WRITE commands to devices A and B. In the middle of the work device B + * received its EXTENDED COPY command EC2 to copy data from device B + * to device A. It also started doing its work in parallel with EC1. + * Then, device A received a serialized or SCSI atomic command, which + * can not proceed until EC1 finished and don't let EC2 generated + * commands to proceed ahead of it. Then, if device B received similar + * serialized or SCSI atomic command, blocking EC2 generated commands, + * we would have a deadlock. + * + * So, we can not allow EXTENDED COPY generated SCSI commands be + * blocked. From other side, we have to honor serialized and SCSI atomic + * commands as well as all other devices blocking events. + * + * To handle it, we, from one side, generate all EXTENDED COPY read and + * writes commands as internal commands, so they bypass blocking + * checking, and from another side, create "fantom" EXTENDED COPY + * commands on all participating devices, then pass "check blocking" on + * all them at once. If any device blocked its command, then undo check + * blocking on all previously processed devices. After the blocked + * command resumed, we retry again. For SCSI atomic commands we check + * each if any of the (fantom) EXTENDED COPY commands conflict with + * existing SCSI atomic commands. If yes, then restart on the conflicting + * (fantom) EXTENDED COPY resume as described above. Then we check all + * newly coming SCSI atomic commands if they conflict with existing + * (fantom) EXTENDED COPY commands. + */ + + if (cmd->internal) { + struct scst_cm_internal_cmd_priv *p = cmd->tgt_i_priv; + + /* cmd is a resumed phantom EXTENDED COPY command */ + + ec_cmd = p->cm_orig_cmd; + + TRACE_BLOCK("Rewaking blocked EC cmd %p (fcmd %p)", + ec_cmd, cmd); + + scst_check_unblock_dev(cmd); + + spin_lock_irq(&ec_cmd->cmd_threads->cmd_list_lock); + list_add_tail(&ec_cmd->cmd_list_entry, + &ec_cmd->cmd_threads->active_cmd_list); + wake_up(&ec_cmd->cmd_threads->cmd_list_waitQ); + spin_unlock_irq(&ec_cmd->cmd_threads->cmd_list_lock); + + res = true; + goto out; + } + + /* cmd is a real ready for exec EXTENDED COPY command */ + + ec_cmd = cmd; + /* + * This could be restart of previously blocked ec_cmd, so + * check unblock it. + */ + TRACE_DBG("Check unblocking ec_cmd %p", ec_cmd); + scst_check_unblock_dev(ec_cmd); + + d = ec_cmd->cmd_data_descriptors; + if (d == NULL) { + spin_lock_bh(&ec_cmd->dev->dev_lock); + res = scst_do_check_blocked_dev(ec_cmd); + spin_unlock_bh(&ec_cmd->dev->dev_lock); + goto out; + } + + local_bh_disable(); + +#if !defined(__CHECKER__) + list_for_each_entry(e, &d->cm_sorted_devs_list, cm_sorted_devs_list_entry) { + spin_lock(&e->cm_fcmd->dev->dev_lock); + } +#endif + + list_for_each_entry(e, &d->cm_sorted_devs_list, cm_sorted_devs_list_entry) { + TRACE_DBG("dev %p (fcmd %p)", e->cm_fcmd->dev, e->cm_fcmd); + res = scst_do_check_blocked_dev(e->cm_fcmd); + if (unlikely(res)) { + TRACE_BLOCK("fcmd %p (ec_cmd %p) blocked, undo " + "check blocking devices", e->cm_fcmd, ec_cmd); + break; + } + } + + if (unlikely(res)) { + struct scst_cmd *blocked_cmd = e->cm_fcmd; + list_for_each_entry(e, &d->cm_sorted_devs_list, + cm_sorted_devs_list_entry) { + if (e->cm_fcmd == blocked_cmd) + break; + __scst_check_unblock_dev(e->cm_fcmd); + e->cm_fcmd->state = SCST_CMD_STATE_EXEC_CHECK_BLOCKING; + } + } + +#if !defined(__CHECKER__) + list_for_each_entry_reverse(e, &d->cm_sorted_devs_list, + cm_sorted_devs_list_entry) { + spin_unlock(&e->cm_fcmd->dev->dev_lock); + } +#endif + + local_bh_enable(); + +out: + TRACE_EXIT_RES(res); + return res; +} + +void scst_cm_abort_ec_cmd(struct scst_cmd *ec_cmd) +{ + struct scst_cm_ec_cmd_priv *p = ec_cmd->cmd_data_descriptors; + struct scst_cm_internal_cmd_priv *ip; + unsigned long flags; + + TRACE_ENTRY(); + + EXTRACHECKS_BUG_ON(ec_cmd->cdb[0] != EXTENDED_COPY); + + spin_lock_irqsave(&scst_cm_lock, flags); + + if (p == NULL) + goto out_unlock; + + TRACE_MGMT_DBG("Aborting fantom and internal commands of ec_cmd %p", + ec_cmd); + + list_for_each_entry(ip, &p->cm_internal_cmd_list, + cm_internal_cmd_list_entry) { + struct scst_cmd *c = ip->cm_cmd; + TRACE_MGMT_DBG("Aborting (f)cmd %p", c); + set_bit(SCST_CMD_ABORTED, &c->cmd_flags); + } + +out_unlock: + spin_unlock_irqrestore(&scst_cm_lock, flags); + + TRACE_EXIT(); + return; +} + +static void scst_cm_del_free_list_id(struct scst_cm_list_id *l, bool locked) +{ + unsigned long flags = 0; + + TRACE_ENTRY(); + + TRACE_DBG("Freeing list id %p", l); + +#if !defined(__CHECKER__) + if (!locked) + spin_lock_irqsave(&scst_cm_lock, flags); +#endif + +#ifdef CONFIG_SMP + EXTRACHECKS_BUG_ON(!spin_is_locked(&scst_cm_lock)); +#endif + + list_del(&l->sess_cm_list_id_entry); + +#if !defined(__CHECKER__) + if (!locked) + spin_unlock_irqrestore(&scst_cm_lock, flags); +#endif + + kfree(l); + + TRACE_EXIT(); + return; +} + +static void scst_cm_sched_del_list_id(struct scst_cmd *ec_cmd) +{ + struct scst_session *sess = ec_cmd->sess; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + struct scst_cm_list_id *l = priv->cm_list_id; + unsigned long flags; + + TRACE_ENTRY(); + + spin_lock_irqsave(&scst_cm_lock, flags); + + if (l->cm_list_id_state != SCST_CM_LIST_ID_STATE_DONE) { + /* + * It can happen for preliminary EC command finish, e.g., + * because of local event like UA. + */ + l->cm_can_be_immed_free = 1; + } + + l->cm_done = 1; + + /* + * Barrier to sync with scst_abort_cmd() and + * scst_mgmt_affected_cmds_done() calling scst_cm_free_pending_list_ids(). + * It has nothing common with cm_done set above. Just in case, actually. + */ + smp_rmb(); + if (test_bit(SCST_CMD_ABORTED, &ec_cmd->cmd_flags) || + l->cm_can_be_immed_free) { + TRACE_DBG("List id %p can be immed freed", l); + scst_cm_del_free_list_id(l, true); + spin_unlock_irqrestore(&scst_cm_lock, flags); + goto out; + } + + l->cm_list_id_state = SCST_CM_LIST_ID_STATE_PENDING_FREE; + l->cm_time_to_free = jiffies + SCST_CM_ID_KEEP_TIME; + + spin_unlock_irqrestore(&scst_cm_lock, flags); + + TRACE_DBG("Schedule pending free list id %p", l); + + schedule_delayed_work(&sess->sess_cm_list_id_cleanup_work, + SCST_CM_ID_KEEP_TIME); + +out: + TRACE_EXIT(); + return; +} + +static struct scst_cm_list_id *scst_cm_add_list_id(struct scst_cmd *cmd, + int list_id) +{ + struct scst_cm_list_id *res; + struct scst_session *sess = cmd->sess; + struct scst_cm_list_id *l; + + TRACE_ENTRY(); + + res = kzalloc(sizeof(*res), GFP_KERNEL); + if (res == NULL) { + TRACE(TRACE_OUT_OF_MEM, "Unable to allocate list_id"); + scst_set_busy(cmd); + goto out; + } + + res->cm_lid = list_id; + res->cm_list_id_state = SCST_CM_LIST_ID_STATE_ACTIVE; + + spin_lock_irq(&scst_cm_lock); + + list_for_each_entry(l, &sess->sess_cm_list_id_list, sess_cm_list_id_entry) { + if (l->cm_lid == list_id) { + if (l->cm_list_id_state == SCST_CM_LIST_ID_STATE_PENDING_FREE) { + scst_cm_del_free_list_id(l, true); + break; + } else { + TRACE_DBG("List id %d already exists", list_id); + scst_set_cmd_error(cmd, + SCST_LOAD_SENSE(scst_sense_operation_in_progress)); + goto out_unlock_free; + } + } + } + + TRACE_DBG("Adding list id %p (id %d)", res, list_id); + list_add_tail(&res->sess_cm_list_id_entry, &sess->sess_cm_list_id_list); + + spin_unlock_irq(&scst_cm_lock); + +out: + TRACE_EXIT(); + return res; + +out_unlock_free: + spin_unlock_irq(&scst_cm_lock); + kfree(res); + res = NULL; + goto out; +} + +void sess_cm_list_id_cleanup_work_fn(struct delayed_work *work) +{ + struct scst_session *sess = container_of((struct delayed_work *)work, + struct scst_session, sess_cm_list_id_cleanup_work); + struct scst_cm_list_id *l, *t; + unsigned long cur_time = jiffies; + unsigned long flags; + + TRACE_ENTRY(); + + /* + * We assume that EC commands finish _more or less_ in order, so + * there's no need to scan the whole list each time. + */ + + spin_lock_irqsave(&scst_cm_lock, flags); + list_for_each_entry_safe(l, t, &sess->sess_cm_list_id_list, sess_cm_list_id_entry) { + if (l->cm_list_id_state != SCST_CM_LIST_ID_STATE_PENDING_FREE) + break; + if (time_after_eq(cur_time, l->cm_time_to_free)) + scst_cm_del_free_list_id(l, true); + else { + TRACE_DBG("Reschedule pending free list ids cleanup"); + schedule_delayed_work(&sess->sess_cm_list_id_cleanup_work, + l->cm_time_to_free - cur_time); + } + } + spin_unlock_irqrestore(&scst_cm_lock, flags); + + TRACE_EXIT(); + return; +} + +void scst_cm_free_pending_list_ids(struct scst_session *sess) +{ + struct scst_cm_list_id *l, *t; + + TRACE_ENTRY(); + + cancel_delayed_work_sync(&sess->sess_cm_list_id_cleanup_work); + + spin_lock_irq(&scst_cm_lock); + list_for_each_entry_safe(l, t, &sess->sess_cm_list_id_list, sess_cm_list_id_entry) { + TRACE_DBG("List id %p, state %d", l, l->cm_list_id_state); + if (l->cm_list_id_state == SCST_CM_LIST_ID_STATE_PENDING_FREE) + scst_cm_del_free_list_id(l, true); + } + spin_unlock_irq(&scst_cm_lock); + + TRACE_EXIT(); + return; +} + +static void scst_cm_copy_status(struct scst_cmd *cmd) +{ + ssize_t length = 0; + uint8_t *buf, tbuf[12]; + int list_id; + struct scst_cm_list_id *l; + struct scst_session *sess = cmd->sess; + bool found = false; + + TRACE_ENTRY(); + + list_id = cmd->cdb[2]; + + spin_lock_irq(&scst_cm_lock); + list_for_each_entry(l, &sess->sess_cm_list_id_list, sess_cm_list_id_entry) { + if (l->cm_lid == list_id) { + TRACE_DBG("list id %p found (id %d)", l, list_id); + found = true; + break; + } + } + if (found) { + l->cm_can_be_immed_free = 1; + + memset(tbuf, 0, sizeof(tbuf)); + put_unaligned_be32(8, &tbuf[0]); + if (l->cm_list_id_state == SCST_CM_LIST_ID_STATE_ACTIVE) + tbuf[4] = 0; /* in progress */ + else if (l->cm_status == 0) + tbuf[4] = 1; /* finished, no errors */ + else + tbuf[4] = 2; /* finished with errors */ + put_unaligned_be16(l->cm_segs_processed, &tbuf[5]); + tbuf[7] = 1; /* KBs */ + put_unaligned_be32(l->cm_written_size >> 10, &tbuf[8]); + + if (l->cm_done) + scst_cm_del_free_list_id(l, true); + } + + l = NULL; /* after unlock it can be immediately get dead */ + + spin_unlock_irq(&scst_cm_lock); + + if (!found) + goto out_list_id_not_found; + + length = scst_get_buf_full_sense(cmd, &buf); + if (unlikely(length <= 0)) + goto out; + + length = min_t(int, (int)sizeof(tbuf), length); + + memcpy(buf, tbuf, length); + scst_set_resp_data_len(cmd, length); + + scst_put_buf_full(cmd, buf); + +out: + TRACE_EXIT(); + return; + +out_list_id_not_found: + TRACE_DBG("list_id %d not found", list_id); + scst_set_invalid_field_in_cdb(cmd, 2, 0); + goto out; +} + +static void scst_cm_failed_seg_details(struct scst_cmd *cmd) +{ + ssize_t length = 0; + uint8_t *buf, *tbuf; + int list_id, size; + struct scst_cm_list_id *l; + struct scst_session *sess = cmd->sess; + bool found = false; + + TRACE_ENTRY(); + + list_id = cmd->cdb[2]; + + size = 60 + SCST_SENSE_BUFFERSIZE; + + tbuf = kzalloc(size, GFP_KERNEL); + if (tbuf == NULL) { + TRACE(TRACE_OUT_OF_MEM, "Unable to allocate FAILED SEGMENTS " + "DETAILS buffer (size %d)", size); + goto out_busy; + } + + spin_lock_irq(&scst_cm_lock); + list_for_each_entry(l, &sess->sess_cm_list_id_list, sess_cm_list_id_entry) { + if (l->cm_lid == list_id) { + TRACE_DBG("list id %p found (id %d)", l, list_id); + found = true; + break; + } + } + if (found) { + if (l->cm_list_id_state == SCST_CM_LIST_ID_STATE_ACTIVE) { + found = false; + goto skip; + } + + if ((cmd->bufflen == 0) || + ((l->cm_status == 0) && (cmd->bufflen >= 60)) || + ((l->cm_status != 0) && (cmd->bufflen >= 60 + SCST_SENSE_BUFFERSIZE))) + l->cm_can_be_immed_free = 1; + + if (l->cm_status == 0) + size = 60; + else + size = 60 + l->cm_sense_len; + + TRACE_DBG("l %p, status %d, sense_len %d, size %d", l, + l->cm_status, l->cm_sense_len, size); + + put_unaligned_be32(size-3, &tbuf[0]); + tbuf[56] = l->cm_status; + EXTRACHECKS_BUG_ON(l->cm_sense_len > SCST_SENSE_BUFFERSIZE); + BUILD_BUG_ON(sizeof(l->cm_sense) != SCST_SENSE_BUFFERSIZE); + put_unaligned_be16(l->cm_sense_len, &tbuf[58]); + if (l->cm_sense_len > 0) + memcpy(&tbuf[60], l->cm_sense, l->cm_sense_len); + + if (l->cm_can_be_immed_free && l->cm_done) + scst_cm_del_free_list_id(l, true); + } + +skip: + l = NULL; /* after unlock it can be immediately get dead */ + + spin_unlock_irq(&scst_cm_lock); + + if (!found) + goto out_list_id_not_found; + + length = scst_get_buf_full_sense(cmd, &buf); + if (unlikely(length <= 0)) + goto out_free; + + length = min_t(int, size, length); + + memcpy(buf, tbuf, length); + scst_set_resp_data_len(cmd, length); + + scst_put_buf_full(cmd, buf); + +out_free: + kfree(tbuf); + + TRACE_EXIT(); + return; + +out_list_id_not_found: + TRACE_DBG("list_id %d not found", list_id); + scst_set_invalid_field_in_cdb(cmd, 2, 0); + goto out_free; + +out_busy: + scst_set_busy(cmd); + goto out_free; +} + +static void scst_cm_oper_parameters(struct scst_cmd *cmd) +{ + ssize_t length = 0; + uint8_t *buf, tbuf[44+2] /* 2 descriptors implemented */; + + TRACE_ENTRY(); + + memset(tbuf, 0, sizeof(tbuf)); + + /* AVAILABLE DATA */ + put_unaligned_be32(sizeof(tbuf) - 4, &tbuf[0]); + + /* SNLID */ + tbuf[4] = 1; + + /* MAXIMUM TARGET DESCRIPTOR COUNT */ + put_unaligned_be16(SCST_CM_MAX_TGT_DESCR_CNT, &tbuf[8]); + + /* MAXIMUM SEGMENT DESCRIPTOR COUNT */ + put_unaligned_be16(((PAGE_SIZE * 2) - sizeof(struct scst_cm_ec_cmd_priv)) / + sizeof(struct scst_ext_copy_seg_descr), &tbuf[10]); + + /* MAXIMUM DESCRIPTOR LIST LENGTH */ + put_unaligned_be32(0xFFFF, &tbuf[12]); + + /* MAXIMUM SEGMENT LENGTH: 256MB */ + put_unaligned_be32(256*1024*1024, &tbuf[16]); + + /* No inline and held data. No stream device max data size. */ + + /* TOTAL CONCURRENT COPIES */ + put_unaligned_be16(0xFFFF, &tbuf[34]); + + /* MAXIMUM CONCURRENT COPIES */ + tbuf[36] = 0xFF; + + /* DATA SEGMENT GRANULARITY */ + tbuf[37] = 16; /* 64K */ + + /* 2 descriptor codes implemented */ + tbuf[43] = 2; + + /* Implemented descriptor codes */ + tbuf[44] = 2; + tbuf[45] = 0xE4; + + length = scst_get_buf_full_sense(cmd, &buf); + if (unlikely(length <= 0)) + goto out; + + length = min_t(int, (int)sizeof(tbuf), length); + + memcpy(buf, tbuf, length); + scst_set_resp_data_len(cmd, length); + + scst_put_buf_full(cmd, buf); + +out: + TRACE_EXIT(); + return; +} + +int scst_cm_rcv_copy_res_exec(struct scst_cmd *cmd) +{ + int res = SCST_EXEC_COMPLETED, action; + + TRACE_ENTRY(); + + action = cmd->cdb[1] & 0x1F; + + switch (action) { + case 0: /* copy status */ + scst_cm_copy_status(cmd); + break; + case 3: /* operational parameters */ + scst_cm_oper_parameters(cmd); + break; + case 4: /* failed segment details */ + scst_cm_failed_seg_details(cmd); + break; + default: + TRACE(TRACE_MINOR, "%s: action %d not supported", cmd->op_name, + action); + scst_set_invalid_field_in_cdb(cmd, 1, + SCST_INVAL_FIELD_BIT_OFFS_VALID | 0); + break; + } + + cmd->completed = 1; /* for success */ + cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD); + + TRACE_EXIT_RES(res); + return res; +} + +struct scst_cm_init_inq_priv { + /* Must be the first for scst_finish_internal_cmd()! */ + scst_i_finish_fn_t cm_init_inq_finish_fn; + + struct scst_device *dev; +}; + +static int scst_cm_send_init_inquiry(struct scst_device *dev, + unsigned int unpacked_lun, struct scst_cm_init_inq_priv *priv); + +static void scst_cm_inq_retry_fn(struct scst_cmd *cmd) +{ + struct scst_cm_init_inq_priv *priv = cmd->tgt_i_priv; + + TRACE_ENTRY(); + + /* cmd->dev can be NULL here! */ + + scst_cm_send_init_inquiry(priv->dev, cmd->lun, priv); + + TRACE_EXIT(); + return; +} + +static void scst_cm_init_inq_finish(struct scst_cmd *cmd) +{ + int length, page_len, off, rc; + uint8_t *buf; + struct scst_cm_init_inq_priv *priv = cmd->tgt_i_priv; + struct scst_device *dev = priv->dev; + + TRACE_ENTRY(); + + /* cmd->dev can be NULL here! */ + + rc = scst_cm_err_check_retry(cmd, cmd->start_time, scst_cm_inq_retry_fn); + if (rc == SCST_CM_STATUS_RETRY) + goto out; + + spin_lock_bh(&dev->dev_lock); + scst_unblock_dev(dev); + spin_unlock_bh(&dev->dev_lock); + + kfree(priv); + priv = NULL; + cmd->tgt_i_priv = NULL; + + if (rc != SCST_CM_STATUS_CMD_SUCCEEDED) { + PRINT_CRIT_ERROR("Unable to perform initial INQUIRY for device " + "%s. Copy manager for this device will be disabled", + dev->virt_name); + goto out; + } + + length = scst_get_buf_full(cmd, &buf); + TRACE_DBG("length %d", length); + if (unlikely(length <= 0)) { + if (length < 0) + PRINT_ERROR("scst_get_buf_full() failed: %d", length); + goto out; + } + + TRACE_BUFF_FLAG(TRACE_DEBUG, "buf", buf, length); + + if (buf[0] != 0) { + TRACE(TRACE_MINOR, "Not supported dev type %x, ignoring", buf[0]); + goto out_put; + } + + if (buf[1] != 0x83) { + PRINT_WARNING("Incorrect page code %x, ignoring", buf[1]); + goto out_put; + } + + page_len = get_unaligned_be16(&buf[2]); + if (page_len+3 > cmd->resp_data_len) { + PRINT_WARNING("Page len (%d) doesn't match resp len (%d), ignoring", + page_len+3, cmd->resp_data_len); + goto out_put; + } + + off = 4; + while (off < page_len) { + int des_len, des_alloc_len; + struct scst_cm_desig *des; + + if (off + 3 >= page_len) { + PRINT_WARNING("Too small page len %d, (off %d), ignoring", + page_len, off); + goto out_put; + } + + des_len = buf[off + 3]; + if ((off + des_len) > page_len) { + PRINT_WARNING("Too small buf len %d (off %d, des_len %d), " + "ignoring", page_len, off, des_len); + goto out_put; + } + + des_len += 4; + + if (((buf[off] & 0xF0) != 0) || ((buf[off+1] & 0xF0) != 0)) { + TRACE_DBG("Unsupported designator (%x, %x), " + "ignoring", buf[off] & 0xF0, buf[off+1] & 0xF0); + goto next; + } + + des_alloc_len = sizeof(*des) + des_len; + des = kzalloc(des_alloc_len, GFP_KERNEL); + if (des == NULL) { + PRINT_CRIT_ERROR("Unable to allocate designator (len %d, " + "type %x), ignoring it", des_alloc_len, + buf[off+1] & 0xF); + goto out_put; + } + + des->desig_tgt_dev = cmd->tgt_dev; + + des->desig_len = des_len; + memcpy(des->desig, &buf[off], des_len); + + TRACE_DBG("des %p, len %d", des, des->desig_len); + TRACE_BUFF_FLAG(TRACE_DEBUG, "des", des->desig, des->desig_len); + + mutex_lock(&scst_cm_mutex); + list_add_tail(&des->cm_desig_list_entry, &scst_cm_desig_list); + mutex_unlock(&scst_cm_mutex); +next: + off += des_len; + TRACE_DBG("off %d", off); + } + +out_put: + scst_put_buf_full(cmd, buf); + +out: + TRACE_EXIT(); + return; +} + +static int scst_cm_send_init_inquiry(struct scst_device *dev, + unsigned int unpacked_lun, struct scst_cm_init_inq_priv *priv) +{ + int res; + static const uint8_t inq_cdb[6] = { INQUIRY, 1, 0x83, 0x10, 0, 0 }; + __be64 lun; + struct scst_cmd *cmd; + + TRACE_ENTRY(); + + if (priv == NULL) { + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (priv == NULL) { + PRINT_ERROR("Unable to alloc priv"); + res = -ENOMEM; + goto out; + } + priv->cm_init_inq_finish_fn = scst_cm_init_inq_finish; + priv->dev = dev; + } + + lun = scst_pack_lun(unpacked_lun, scst_cm_sess->acg->addr_method); + + cmd = scst_rx_cmd(scst_cm_sess, (const uint8_t *)&lun, + sizeof(lun), inq_cdb, sizeof(inq_cdb), false); + if (cmd == NULL) { + res = -ENOMEM; + goto out_free; + } + + scst_cmd_set_expected(cmd, SCST_DATA_READ, 4096); + scst_cmd_set_queue_type(cmd, SCST_CMD_QUEUE_HEAD_OF_QUEUE); + scst_cmd_set_tgt_priv(cmd, priv); + + cmd->bypass_blocking = 1; + + scst_cmd_init_done(cmd, SCST_CONTEXT_THREAD); + + res = 0; + +out: + TRACE_EXIT_RES(res); + return res; + +out_free: + kfree(priv); + goto out; +} + +/* scst_mutex supposed to be held and activities suspended */ +static bool scst_cm_is_lun_free(unsigned int lun) +{ + bool res = true; + struct list_head *head = &scst_cm_sess->sess_tgt_dev_list[SESS_TGT_DEV_LIST_HASH_FN(lun)]; + struct scst_tgt_dev *tgt_dev; + + TRACE_ENTRY(); + + list_for_each_entry(tgt_dev, head, sess_tgt_dev_list_entry) { + if (tgt_dev->lun == lun) { + res = false; + break; + } + } + + TRACE_EXIT_RES(res); + return res; +} + +/* scst_mutex supposed to be held and activities suspended */ +static int scst_cm_dev_register(struct scst_device *dev) +{ + int res, i; + unsigned int lun; + struct scst_acg_dev *acg_dev; + + TRACE_ENTRY(); + + for (i = 0; i < SESS_TGT_DEV_LIST_HASH_SIZE; i++) { + struct scst_tgt_dev *tgt_dev; + struct list_head *head = &scst_cm_sess->sess_tgt_dev_list[i]; + list_for_each_entry(tgt_dev, head, sess_tgt_dev_list_entry) { + if (tgt_dev->dev == dev) { + PRINT_ERROR("Copy Manager already registered " + "device %s", dev->virt_name); + res = -EEXIST; + goto out; + } + } + } + + while (1) { + lun = scst_cm_next_lun++; + if (lun == SCST_MAX_LUN) + continue; + if (scst_cm_is_lun_free(lun)) + break; + }; + + res = scst_acg_add_lun(scst_cm_tgt->default_acg, + scst_cm_tgt->tgt_luns_kobj, dev, lun, false, false, &acg_dev); + if (res != 0) + goto out_err; + + spin_lock_bh(&dev->dev_lock); + scst_block_dev(dev); + spin_unlock_bh(&dev->dev_lock); + + res = scst_cm_send_init_inquiry(dev, lun, NULL); + if (res != 0) + goto out_unblock; + +out: + TRACE_EXIT_RES(res); + return res; + +out_unblock: + spin_lock_bh(&dev->dev_lock); + scst_unblock_dev(dev); + spin_unlock_bh(&dev->dev_lock); + scst_acg_del_lun(scst_cm_tgt->default_acg, lun, false); + +out_err: + scst_cm_next_lun--; + goto out; +} + +/* scst_mutex supposed to be held and activities suspended */ +static void scst_cm_dev_unregister(struct scst_device *dev) +{ + int i; + struct scst_cm_desig *des, *t; + + TRACE_ENTRY(); + + list_for_each_entry_safe(des, t, &scst_cm_desig_list, cm_desig_list_entry) { + if (des->desig_tgt_dev->dev == dev) { + TRACE_DBG("Deleting des %p", des); + list_del(&des->cm_desig_list_entry); + kfree(des); + } + } + + for (i = 0; i < SESS_TGT_DEV_LIST_HASH_SIZE; i++) { + struct scst_tgt_dev *tgt_dev; + struct list_head *head = &scst_cm_sess->sess_tgt_dev_list[i]; + list_for_each_entry(tgt_dev, head, sess_tgt_dev_list_entry) { + if (tgt_dev->dev == dev) { + scst_acg_del_lun(scst_cm_tgt->default_acg, + tgt_dev->lun, false); + break; + } + } + } + + TRACE_EXIT(); + return; +} + +/* scst_mutex supposed to be held and activities suspended */ +int scst_cm_on_dev_register(struct scst_device *dev) +{ + int res = 0; + + TRACE_ENTRY(); + + if (!dev->handler->auto_cm_assignment_possible) + goto out; + + res = scst_cm_dev_register(dev); + +out: + TRACE_EXIT_RES(res); + return res; +} + +/* scst_mutex supposed to be held and activities suspended */ +void scst_cm_on_dev_unregister(struct scst_device *dev) +{ + TRACE_ENTRY(); + + scst_cm_dev_unregister(dev); + + TRACE_EXIT(); + return; +} + +/* scst_mutex supposed to be locked */ +static bool scst_cm_check_access_acg(const char *initiator_name, + const struct scst_device *dev, const struct scst_acg *acg, + bool default_acg) +{ + bool res = true; + struct scst_acg_dev *acg_dev; + + TRACE_ENTRY(); + + list_for_each_entry(acg_dev, &acg->acg_dev_list, acg_dev_list_entry) { + if (acg_dev->dev == dev) { + struct scst_acn *acn; + if (default_acg) + goto found; + list_for_each_entry(acn, &acg->acn_list, acn_list_entry) { + if (strcmp(acn->name, initiator_name) == 0) + goto found; + } + } + } + + res = false; + +found: + TRACE_EXIT_RES(res); + return res; +} + +static bool scst_cm_check_access(const char *initiator_name, + const struct scst_device *dev, bool *read_only) +{ + bool res = true; + struct scst_tgt_template *tgtt; + + TRACE_ENTRY(); + + if (scst_cm_allow_not_connected_copy) + goto out_rd_only; + + /* ToDo: make it hash based */ + + /* + * We can't use scst_mutex on commands processing path, because + * otherwise we can fall in a deadlock with kthread_stop() in + * scst_del_threads() waiting for this command to finish. + */ + mutex_lock(&scst_mutex2); + + list_for_each_entry(tgtt, &scst_template_list, scst_template_list_entry) { + struct scst_tgt *tgt; + + list_for_each_entry(tgt, &tgtt->tgt_list, tgt_list_entry) { + struct scst_acg *acg; + + if (tgt == scst_cm_tgt) + continue; + + TRACE_DBG("Checking tgt %s", tgt->tgt_name); + + if (scst_cm_check_access_acg(initiator_name, dev, tgt->default_acg, true)) + goto out_unlock_rd_only; + + list_for_each_entry(acg, &tgt->tgt_acg_list, acg_list_entry) { + if (scst_cm_check_access_acg(initiator_name, dev, acg, false)) + goto out_unlock_rd_only; + } + } + } + + res = false; + PRINT_WARNING("Initiator %s not allowed to use device %s in EXTENDED " + "COPY command", initiator_name, dev->virt_name); + +out_unlock_rd_only: + mutex_unlock(&scst_mutex2); + +out_rd_only: + *read_only = dev->dev_rd_only || dev->swp; + + TRACE_EXIT_RES(res); + return res; +} + +struct scst_cm_tgt_descr { + struct scst_tgt_dev *tgt_dev; + unsigned int read_only:1; + int param_offs; +}; + +static int scst_cm_parse_id_tgt_descr(struct scst_cmd *cmd, const uint8_t *seg, + int offs, struct scst_cm_tgt_descr *tgt_descr) +{ + int res = 32; + struct scst_cm_desig *des; + int block; + bool read_only = false; + + TRACE_ENTRY(); + + TRACE_BUFF_FLAG(TRACE_DEBUG, "seg", seg, 32); + + EXTRACHECKS_BUG_ON(seg[0] != 0xE4); + + if ((seg[1] & 0xC0) != 0) { + PRINT_WARNING("LU ID %x not supported", seg[1] & 0xC0); + scst_set_invalid_field_in_parm_list(cmd, offs+1, + SCST_INVAL_FIELD_BIT_OFFS_VALID | 6); + goto out_err; + } + + if ((seg[1] & 0x20) != 0) { + TRACE_DBG("NULL tgt descriptor"); + tgt_descr->tgt_dev = NULL; + goto out; + } + + if ((seg[1] & 0xF) != 0) { + PRINT_WARNING("PERIPHERAL DEVICE TYPE %d not supported", + seg[1] & 0xF); + scst_set_invalid_field_in_parm_list(cmd, offs+1, + SCST_INVAL_FIELD_BIT_OFFS_VALID | 0); + goto out_err; + } + + block = get_unaligned_be24(&seg[29]); + + mutex_lock(&scst_cm_mutex); + + /* ToDo: make it hash based */ + + list_for_each_entry(des, &scst_cm_desig_list, cm_desig_list_entry) { + TRACE_DBG("des %p (tgt_dev %p, lun %lld)", des, des->desig_tgt_dev, + (unsigned long long)des->desig_tgt_dev->lun); + if (seg[4] != des->desig[0]) + continue; + if (seg[5] != des->desig[1]) + continue; + if (seg[7] > des->desig[3]) + continue; + if (memcmp(&des->desig[4], &seg[8], min_t(int, seg[7], des->desig[3])) == 0) { + TRACE_DBG("Tgt_dev %p (lun %lld) found", + des->desig_tgt_dev, + (unsigned long long)des->desig_tgt_dev->lun); + + mutex_unlock(&scst_cm_mutex); + + if (block != des->desig_tgt_dev->dev->block_size) { + PRINT_WARNING("Block size %d doesn't match %d", block, + des->desig_tgt_dev->dev->block_size); + scst_set_invalid_field_in_parm_list(cmd, offs+29, 0); + goto out_err; + } + + if (!scst_cm_check_access(cmd->sess->initiator_name, + des->desig_tgt_dev->dev, &read_only)) + goto out_not_found; + + tgt_descr->tgt_dev = des->desig_tgt_dev; + tgt_descr->read_only = read_only; + TRACE_DBG("Found des %p (tgt_dev %p, read_only %d)", + des, tgt_descr->tgt_dev, tgt_descr->read_only); + goto out; + } + } + + mutex_unlock(&scst_cm_mutex); + + TRACE(TRACE_MINOR|TRACE_SCSI, "Target descriptor designator not found " + "(initiator %s, offs %d)", cmd->sess->initiator_name, offs); + TRACE_BUFF_FLAG(TRACE_MINOR|TRACE_SCSI, "Designator", seg, 32); + +out_not_found: + scst_set_invalid_field_in_parm_list(cmd, offs, 0); + +out_err: + res = -1; + +out: + TRACE_EXIT_RES(res); + return res; +} + +static int scst_cm_set_seg_err_sense(struct scst_cmd *cmd, int asc, int ascq, + int seg_num, int offs) +{ + int res, d_sense = scst_get_cmd_dev_d_sense(cmd); + + TRACE_ENTRY(); + + TRACE_DBG("cmd %p, seg %d, offs %d (d_sense %d)", cmd, seg_num, offs, + d_sense); + + res = scst_set_cmd_error_status(cmd, SAM_STAT_CHECK_CONDITION); + if (res != 0) + goto out; + + res = scst_alloc_sense(cmd, 1); + if (res != 0) { + PRINT_ERROR("Lost COPY ABORTED sense data"); + goto out; + } + + sBUG_ON(cmd->sense_buflen < 18); + BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < 18); + + if (d_sense) { + /* Descriptor format */ + cmd->sense[0] = 0x72; + cmd->sense[1] = COPY_ABORTED; + cmd->sense[2] = asc; + cmd->sense[3] = ascq; + cmd->sense[7] = 20; /* additional Sense Length */ + + cmd->sense[8] = 1; /* Command specific descriptor */ + cmd->sense[9] = 0x0A; + put_unaligned_be16(seg_num, &cmd->sense[14]); + + cmd->sense[20] = 2; /* Sense key specific descriptor */ + cmd->sense[21] = 6; + cmd->sense[24] = 0xA0; + put_unaligned_be16(offs, &cmd->sense[25]); + + cmd->sense_valid_len = 28; + } else { + /* Fixed format */ + cmd->sense[0] = 0x70; + cmd->sense[2] = COPY_ABORTED; + cmd->sense[7] = 0x0a; /* additional Sense Length */ + cmd->sense[12] = asc; + cmd->sense[13] = ascq; + + put_unaligned_be16(seg_num, &cmd->sense[10]); + + cmd->sense[15] = 0xA0; + put_unaligned_be16(offs, &cmd->sense[16]); + + cmd->sense_valid_len = 18; + } + + TRACE_BUFFER("Sense set", cmd->sense, cmd->sense_valid_len); + +out: + TRACE_EXIT_RES(res); + return res; +} + +static void scst_cm_fantom_cmd_finished(struct scst_cmd *cmd) +{ + TRACE_ENTRY(); + + TRACE_DBG("Fantom cmd %p", cmd); + + EXTRACHECKS_BUG_ON(!cmd->internal); + + /* Nothing to do */ + + TRACE_EXIT(); + return; +} + +static int scst_cm_add_to_descr_list(struct scst_cmd *ec_cmd, + struct scst_tgt_dev *tgt_dev) +{ + int res; + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + struct scst_cm_dev_entry *e, *t; + struct scst_cmd *fcmd; + bool added; + + TRACE_ENTRY(); + + /* Check if we already have this device in the list */ + list_for_each_entry(e, &priv->cm_sorted_devs_list, cm_sorted_devs_list_entry) { + if (e->cm_fcmd->dev == tgt_dev->dev) { + TRACE_DBG("Dev %p is already in cm_sorted_devs_list", + tgt_dev->dev); + goto out_success; + } + } + + e = kzalloc(sizeof(*e), GFP_KERNEL); + if (e == NULL) { + PRINT_ERROR("Unable to allocate scst_cm_dev_entry (size %d)", + (int)sizeof(*e)); + goto out_enomem; + } + + if (ec_cmd->dev == tgt_dev->dev) { + fcmd = ec_cmd; + goto skip_fcmd_create; + } + + fcmd = __scst_create_prepare_internal_cmd(ec_cmd->cdb, + ec_cmd->cdb_len, SCST_CMD_QUEUE_SIMPLE, tgt_dev, + GFP_KERNEL, true); + if (fcmd == NULL) { + goto out_enomem_free_e; + } + + fcmd->expected_data_direction = ec_cmd->expected_data_direction; + fcmd->expected_transfer_len_full = ec_cmd->expected_transfer_len_full; + fcmd->expected_values_set = 1; + + fcmd->cmd_data_descriptors = ec_cmd->cmd_data_descriptors; + fcmd->cmd_data_descriptors_cnt = ec_cmd->cmd_data_descriptors_cnt; + + fcmd->state = SCST_CMD_STATE_EXEC_CHECK_BLOCKING; + + res = scst_cm_add_to_internal_cmd_list(fcmd, ec_cmd, ec_cmd, + scst_cm_fantom_cmd_finished); + if (res != 0) + goto out_free_cmd; + +skip_fcmd_create: + TRACE_DBG("ec_cmd %p, e %p, fcmd %p, tgt_dev %p (dev %p)", ec_cmd, e, fcmd, + tgt_dev, tgt_dev->dev); + + e->cm_fcmd = fcmd; + + added = false; + list_for_each_entry_reverse(t, &priv->cm_sorted_devs_list, cm_sorted_devs_list_entry) { + EXTRACHECKS_BUG_ON(t->cm_fcmd->dev == tgt_dev->dev); + if (((unsigned long)e->cm_fcmd->dev) > ((unsigned long)t->cm_fcmd->dev)) { + __list_add(&e->cm_sorted_devs_list_entry, + &t->cm_sorted_devs_list_entry, + t->cm_sorted_devs_list_entry.next); + added = true; + break; + } + } + if (!added) + list_add(&e->cm_sorted_devs_list_entry, + &priv->cm_sorted_devs_list); + +#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_EXTRACHECKS) + { + struct scst_cm_dev_entry *tp = NULL; + list_for_each_entry(t, &priv->cm_sorted_devs_list, cm_sorted_devs_list_entry) { + TRACE_DBG("t %p, cm dev %p", t, t->cm_fcmd->dev); + if (tp != NULL) { + if (((unsigned long)t->cm_fcmd->dev) <= ((unsigned long)tp->cm_fcmd->dev)) { + list_for_each_entry(t, &priv->cm_sorted_devs_list, cm_sorted_devs_list_entry) { + printk(KERN_EMERG "%s: t %p, cm dev %p\n", __func__, + t, t->cm_fcmd->dev); + } + sBUG(); + break; + } + tp = t; + } + tp = t; + } + } +#endif + +out_success: + res = 0; + +out: + TRACE_EXIT_RES(res); + return res; + +out_free_cmd: + __scst_cmd_put(fcmd); + +out_enomem_free_e: + kfree(e); + +out_enomem: + scst_set_busy(ec_cmd); + res = -ENOMEM; + goto out; +} + +static int scst_cm_parse_b2b_seg_descr(struct scst_cmd *ec_cmd, + const uint8_t *seg, const struct scst_cm_tgt_descr *tgt_descrs, + int tgt_descrs_cnt, int seg_num) +{ + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + struct scst_ext_copy_seg_descr *d = &priv->cm_seg_descrs[seg_num]; + int res = 28, rc, len, src_des_idx, tgt_des_idx, blocks, dc; + const struct scst_cm_tgt_descr *src_des, *tgt_des; + + TRACE_ENTRY(); + + TRACE_BUFF_FLAG(TRACE_DEBUG, "seg", seg, 28); + + EXTRACHECKS_BUG_ON(seg[0] != 2); + + len = get_unaligned_be16(&seg[2]); + if (len != 0x18) { + PRINT_WARNING("Incorrect B2B segment descriptor len %d", len); + scst_cm_set_seg_err_sense(ec_cmd, 0, 0, seg_num, 2); + goto out_err; + } + + src_des_idx = get_unaligned_be16(&seg[4]); + if (src_des_idx >= tgt_descrs_cnt) { + PRINT_WARNING("Invalid src descriptor index %d", src_des_idx); + scst_cm_set_seg_err_sense(ec_cmd, 0, 0, seg_num, 4); + goto out_err; + } + + src_des = &tgt_descrs[src_des_idx]; + if (src_des->tgt_dev == NULL) { + PRINT_WARNING("Segment with NULL src tgt device"); + /* COPY TARGET DEVICE NOT REACHABLE */ + scst_cm_set_seg_err_sense(ec_cmd, 0xD, 2, seg_num, 4); + goto out_err; + } + + tgt_des_idx = get_unaligned_be16(&seg[6]); + if (tgt_des_idx >= tgt_descrs_cnt) { + PRINT_WARNING("Invalid tgt descriptor index %d", tgt_des_idx); + scst_cm_set_seg_err_sense(ec_cmd, 0, 0, seg_num, 6); + goto out_err; + } + + tgt_des = &tgt_descrs[tgt_des_idx]; + if (tgt_des->tgt_dev == NULL) { + PRINT_WARNING("Segment with NULL tgt device"); + /* COPY TARGET DEVICE NOT REACHABLE */ + scst_cm_set_seg_err_sense(ec_cmd, 0xD, 2, seg_num, 6); + goto out_err; + } + if (tgt_des->read_only){ + PRINT_WARNING("Target descriptor refers to read-only device"); + scst_cm_set_seg_err_sense(ec_cmd, 0, 0, seg_num, 6); + goto out_err; + } + + dc = (seg[1] >> 1) & 1; + blocks = get_unaligned_be16(&seg[10]); + if (dc) + len = blocks << tgt_des->tgt_dev->dev->block_shift; + else + len = blocks << src_des->tgt_dev->dev->block_shift; + + if (unlikely((len & (src_des->tgt_dev->dev->block_size-1)) != 0) || + unlikely((len & (tgt_des->tgt_dev->dev->block_size-1)) != 0)) { + PRINT_WARNING("Data len %d is not even for block size (src block " + "size %d, dst block size %d)", len, + src_des->tgt_dev->dev->block_size, + tgt_des->tgt_dev->dev->block_size); + scst_cm_set_seg_err_sense(ec_cmd, 0, 0, seg_num, 10); + goto out_err; + } + + d->type = SCST_EXT_COPY_SEG_DATA; + d->data_descr.data_len = len; + d->src_tgt_dev = src_des->tgt_dev; + d->data_descr.src_lba = get_unaligned_be64(&seg[12]); + d->dst_tgt_dev = tgt_des->tgt_dev; + d->data_descr.dst_lba = get_unaligned_be64(&seg[20]); + d->tgt_descr_offs = tgt_des->param_offs; + + TRACE(TRACE_DEBUG|TRACE_SCSI, "ec_cmd %p, src dev %s, dst dev %s, " + "len %d, src_lba %lld, dst_lba %lld", ec_cmd, + d->src_tgt_dev->dev->virt_name, d->dst_tgt_dev->dev->virt_name, + len, (long long)d->data_descr.src_lba, + (long long)d->data_descr.dst_lba); + + TRACE_DBG("src tgt_dev %p, dst tgt_dev %p, tgt_descr_offs %d", + d->src_tgt_dev, d->dst_tgt_dev, d->tgt_descr_offs); + + rc = scst_cm_add_to_descr_list(ec_cmd, src_des->tgt_dev); + if (rc != 0) { + res = rc; + goto out; + } + + rc = scst_cm_add_to_descr_list(ec_cmd, tgt_des->tgt_dev); + if (rc != 0) { + res = rc; + goto out; + } + +out: + TRACE_EXIT_RES(res); + return res; + +out_err: + res = -1; + goto out; +} + +static void scst_cm_free_ec_priv(struct scst_cmd *ec_cmd, bool unblock_dev) +{ + struct scst_cm_ec_cmd_priv *p = ec_cmd->cmd_data_descriptors; + struct scst_cm_dev_entry *e, *t; + struct scst_cm_internal_cmd_priv *ip, *it; + unsigned long flags; + + TRACE_ENTRY(); + + list_for_each_entry_safe(e, t, &p->cm_sorted_devs_list, + cm_sorted_devs_list_entry) { + TRACE_DBG("Deleting e %p", e); + list_del(&e->cm_sorted_devs_list_entry); + kfree(e); + } + + list_for_each_entry_safe(ip, it, &p->cm_internal_cmd_list, + cm_internal_cmd_list_entry) { + struct scst_cmd *c = ip->cm_cmd; + scst_cm_del_free_from_internal_cmd_list(c, unblock_dev); + __scst_cmd_put(c); + } + + /* Lock to sync with scst_cm_abort_ec_cmd() */ + spin_lock_irqsave(&scst_cm_lock, flags); + ec_cmd->cmd_data_descriptors = NULL; + ec_cmd->cmd_data_descriptors_cnt = 0; + spin_unlock_irqrestore(&scst_cm_lock, flags); + + kfree(p); + + TRACE_EXIT(); + return; +} + +int scst_cm_parse_descriptors(struct scst_cmd *ec_cmd) +{ + int res = 0, rc; + struct scst_cm_list_id *plist_id = NULL; + ssize_t length = 0; + uint8_t *buf; + int list_id, list_id_usage, len, tgt_len, seg_len; + struct scst_cm_ec_cmd_priv *p; + int tgt_cnt, seg_cnt, i, offs, t; + struct scst_cm_tgt_descr *tgt_descrs; + + TRACE_ENTRY(); + + EXTRACHECKS_BUG_ON(ec_cmd->cmd_data_descriptors != NULL); + + length = scst_get_buf_full_sense(ec_cmd, &buf); + if (unlikely(length <= 0)) { + if (length == 0) + goto out_put; + else + goto out_abn; + } + + if (length < 16) { + PRINT_WARNING("Too small EXTENDED COPY data len %d", (int)length); + scst_set_invalid_field_in_cdb(ec_cmd, 10, 0); + goto out_abn_put; + } + + list_id = buf[0]; + list_id_usage = (buf[1] & 0x18) >> 3; + + TRACE_BUFF_FLAG(TRACE_DEBUG, "buf", buf, length); + + TRACE_DBG("list_id %d, list_id_usage %d", list_id, list_id_usage); + + switch (list_id_usage) { + case 0: + case 2: + plist_id = scst_cm_add_list_id(ec_cmd, list_id); + if (plist_id == NULL) + goto out_abn_put; + break; + case 3: + if (list_id != 0) { + PRINT_WARNING("Invalid list ID %d with list ID usage %d", + list_id, list_id_usage); + scst_set_invalid_field_in_parm_list(ec_cmd, 0, 0); + goto out_abn_put; + } + break; + default: + PRINT_WARNING("Invalid list ID usage %d, rejecting", list_id_usage); + scst_set_invalid_field_in_parm_list(ec_cmd, 1, + SCST_INVAL_FIELD_BIT_OFFS_VALID | 3); + goto out_abn_put; + } + + len = get_unaligned_be32(&buf[12]); + if (len != 0) { + PRINT_WARNING("Inline data not supported (len %d)", len); + scst_set_cmd_error(ec_cmd, SCST_LOAD_SENSE(scst_sense_inline_data_length_exceeded)); + goto out_del_abn_put; + } + + tgt_len = get_unaligned_be16(&buf[2]); + seg_len = get_unaligned_be32(&buf[8]); + + if (tgt_len == 0) { + if (seg_len == 0) + goto out_del_put; + else { + PRINT_WARNING("Zero target descriptors with none zero " + "segments len (%d)", seg_len); + scst_set_invalid_field_in_parm_list(ec_cmd, 2, 0); + goto out_del_abn_put; + } + } + + if ((tgt_len + seg_len + 16) > length) { + PRINT_WARNING("Parameters truncation"); + scst_set_cmd_error(ec_cmd, + SCST_LOAD_SENSE(scst_sense_parameter_list_length_invalid)); + goto out_del_abn_put; + } + + if ((tgt_len + seg_len + 16) != length) { + PRINT_WARNING("Unexpected inline data"); + scst_set_cmd_error(ec_cmd, + SCST_LOAD_SENSE(scst_sense_inline_data_length_exceeded)); + goto out_del_abn_put; + } + + if ((tgt_len % 32) != 0) { + PRINT_WARNING("Invalid tgt len %d", tgt_len); + scst_set_invalid_field_in_parm_list(ec_cmd, 2, 0); + goto out_del_abn_put; + } + + tgt_cnt = tgt_len/32; + if (tgt_cnt > SCST_CM_MAX_TGT_DESCR_CNT) { + PRINT_WARNING("Too many target descriptors %d", tgt_cnt); + scst_set_cmd_error(ec_cmd, + SCST_LOAD_SENSE(scst_sense_too_many_target_descriptors)); + goto out_del_abn_put; + } + + TRACE_DBG("tgt_cnt %d", tgt_cnt); + + tgt_descrs = kzalloc(sizeof(*tgt_descrs) * tgt_cnt, GFP_KERNEL); + if (tgt_descrs == NULL) { + TRACE(TRACE_OUT_OF_MEM, "Unable to allocate tgt_descrs " + "(count %d, size %zd)", tgt_cnt, + sizeof(*tgt_descrs) * tgt_cnt); + scst_set_busy(ec_cmd); + goto out_del_abn_put; + } + + offs = 16; + for (i = 0; i < tgt_cnt; i++) { + TRACE_DBG("offs %d", offs); + switch (buf[offs]) { + case 0xE4: /* identification descriptor target descriptor format */ + rc = scst_cm_parse_id_tgt_descr(ec_cmd, &buf[offs], offs, + &tgt_descrs[i]); + if (rc <= 0) + goto out_free_tgt_descr; + break; + default: + PRINT_WARNING("Not supported target descriptor %x", buf[offs]); + scst_set_cmd_error(ec_cmd, SCST_LOAD_SENSE(scst_sense_unsupported_tgt_descr_type)); + goto out_free_tgt_descr; + } + tgt_descrs[i].param_offs = offs; + offs += rc; + } + + WARN_ON(offs != tgt_len + 16); + + t = offs; + seg_cnt = 0; + while (offs < length) { + switch (buf[offs]) { + case 2: /* block device to block device segment descriptor */ + offs += 28; + break; + default: + PRINT_WARNING("Not supported segment descriptor %x", buf[offs]); + scst_set_cmd_error(ec_cmd, + SCST_LOAD_SENSE(scst_sense_unsupported_seg_descr_type)); + goto out_free_tgt_descr; + } + seg_cnt++; + } + offs = t; + + TRACE_DBG("seg_cnt %d", seg_cnt); + + p = kzalloc(sizeof(*p) + seg_cnt * sizeof(struct scst_ext_copy_seg_descr), GFP_KERNEL); + if (p == NULL) { + TRACE(TRACE_OUT_OF_MEM, "Unable to allocate Extended Copy " + "descriptors (seg_cnt %d)", seg_cnt); + scst_set_busy(ec_cmd); + goto out_free_tgt_descr; + } + + p->cm_list_id = plist_id; + plist_id = NULL; + INIT_LIST_HEAD(&p->cm_sorted_devs_list); + INIT_LIST_HEAD(&p->cm_internal_cmd_list); + p->cm_error = SCST_CM_ERROR_NONE; + mutex_init(&p->cm_mutex); + + ec_cmd->cmd_data_descriptors = p; + ec_cmd->cmd_data_descriptors_cnt = seg_cnt; + + res = scst_cm_add_to_descr_list(ec_cmd, ec_cmd->tgt_dev); + if (res != 0) + goto out_free_p; + + for (i = 0; i < seg_cnt; i++) { + TRACE_DBG("offs %d", offs); + switch (buf[offs]) { + case 2: /* block device to block device segment descriptor */ + rc = scst_cm_parse_b2b_seg_descr(ec_cmd, &buf[offs], + tgt_descrs, tgt_cnt, i); + if (rc <= 0) { + if (rc == -ENOMEM) + goto out_free_p; + else { + /* + * We may need to keep list_id for a + * while for further FAILED SEGMENT + * DETAILS of RECEIVE COPY RESULTS + */ + scst_cm_store_list_id_details(ec_cmd); + goto out_free_tgt_descr; + } + } + EXTRACHECKS_BUG_ON(rc != 28); + break; + default: + sBUG(); + } + offs += rc; + } + + kfree(tgt_descrs); + +out_del_put: + if (plist_id != NULL) + scst_cm_del_free_list_id(plist_id, false); + +out_put: + scst_put_buf_full(ec_cmd, buf); + +out: + TRACE_EXIT_RES(res); + return res; + +out_free_p: + scst_cm_free_ec_priv(ec_cmd, false); + +out_free_tgt_descr: + kfree(tgt_descrs); + +out_del_abn_put: + if (plist_id != NULL) + scst_cm_del_free_list_id(plist_id, false); + +out_abn_put: + scst_put_buf_full(ec_cmd, buf); + +out_abn: + scst_set_cmd_abnormal_done_state(ec_cmd); + res = -1; + goto out; +} + +void scst_cm_free_descriptors(struct scst_cmd *ec_cmd) +{ + struct scst_cm_ec_cmd_priv *priv = ec_cmd->cmd_data_descriptors; + + TRACE_ENTRY(); + + TRACE_DBG("cmd %p (internal %d)", ec_cmd, ec_cmd->internal); + + if (priv == NULL) { + /* It can be for early errors */ + goto out; + } + + if (ec_cmd->internal) + goto out; + + if (priv->cm_list_id != NULL) + scst_cm_sched_del_list_id(ec_cmd); + + scst_cm_free_ec_priv(ec_cmd, true); + +out: + TRACE_EXIT(); + return; +} + +#ifndef CONFIG_SCST_PROC + +static ssize_t scst_cm_allow_not_conn_copy_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + ssize_t res; + + TRACE_ENTRY(); + + res = sprintf(buf, "%d\n%s", scst_cm_allow_not_connected_copy, + (scst_cm_allow_not_connected_copy == SCST_ALLOW_NOT_CONN_COPY_DEF) ? + "" : SCST_SYSFS_KEY_MARK "\n"); + + TRACE_EXIT_RES(res); + return res; +} + +static ssize_t scst_cm_allow_not_conn_copy_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buffer, size_t size) +{ + ssize_t res; + unsigned long val; + + TRACE_ENTRY(); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39) + res = kstrtoul(buffer, 0, &val); +#else + res = strict_strtoul(buffer, 0, &val); +#endif + if (res != 0) { + PRINT_ERROR("strtoul() for %s failed: %zd", buffer, res); + goto out; + } + + scst_cm_allow_not_connected_copy = val; + + res = size; + +out: + TRACE_EXIT_RES(res); + return res; +} + +static struct kobj_attribute scst_cm_allow_not_conn_copy_attr = + __ATTR(allow_not_connected_copy, S_IRUGO|S_IWUSR, + scst_cm_allow_not_conn_copy_show, + scst_cm_allow_not_conn_copy_store); + +static ssize_t scst_cm_mgmt_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + static const char help[] = + "Usage: echo \"add H:C:I:L\" >mgmt\n" + " echo \"add VNAME\" >mgmt\n" + " echo \"del H:C:I:L\" >mgmt\n" + " echo \"del VNAME\" >mgmt\n"; + + return sprintf(buf, "%s", help); +} + +static int scst_cm_mgmt(struct scst_sysfs_work_item *work) +{ + int res = 0; + char *pp, *action, *devstr; + unsigned int host, channel, id, lun; + char *buf = work->buf; + bool vdev; + struct scst_device *d, *dev = NULL; + + TRACE_ENTRY(); + + TRACE_DBG("buffer %s", buf); + + pp = buf; + action = scst_get_next_lexem(&pp); + devstr = scst_get_next_lexem(&pp); + if (*devstr == '\0') { + PRINT_ERROR("%s", "Device required"); + res = -EINVAL; + goto out; + } + + if (*scst_get_next_lexem(&pp) != '\0') { + PRINT_ERROR("%s", "Too many parameters"); + res = -EINVAL; + goto out_syntax_err; + } + + if (sscanf(devstr, "%u:%u:%u:%u", &host, &channel, &id, &lun) != 4) { + vdev = true; + TRACE_DBG("Virt dev %s", devstr); + } else { + vdev = false; + TRACE_DBG("Pass-through dev %d:%d:%d:%d", host, channel, id, lun); + } + + res = scst_suspend_activity(SCST_SUSPEND_TIMEOUT_USER); + if (res != 0) + goto out; + + res = mutex_lock_interruptible(&scst_mutex); + if (res != 0) + goto out_resume; + + list_for_each_entry(d, &scst_dev_list, dev_list_entry) { + if (vdev) { + if ((d->scsi_dev == NULL) && + (strcmp(d->virt_name, devstr) == 0)) { + dev = d; + break; + } + } else if (d->scsi_dev != NULL && + d->scsi_dev->host->host_no == host && + d->scsi_dev->channel == channel && + d->scsi_dev->id == id && + d->scsi_dev->lun == lun) { + dev = d; + break; + } + } + if (dev == NULL) { + PRINT_ERROR("Device %s not found", devstr); + res = -EINVAL; + goto out_unlock; + } else + TRACE_DBG("Dev %p (%s) found", dev, dev->virt_name); + + if (strcasecmp("add", action) == 0) + res = scst_cm_dev_register(dev); + else if (strcasecmp("del", action) == 0) + scst_cm_dev_unregister(dev); + else { + PRINT_ERROR("Action %s not understood", action); + res = -EINVAL; + } + +out_unlock: + mutex_unlock(&scst_mutex); + +out_resume: + scst_resume_activity(); + +out: + TRACE_EXIT_RES(res); + return res; + +out_syntax_err: + PRINT_ERROR("Syntax error on \"%s\"", buf); + res = -EINVAL; + goto out; +} + +static ssize_t scst_cm_mgmt_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buffer, size_t size) +{ + int res; + struct scst_sysfs_work_item *work; + char *i_buf; + + TRACE_ENTRY(); + + i_buf = kasprintf(GFP_KERNEL, "%.*s", (int)size, buffer); + if (i_buf == NULL) { + PRINT_ERROR("Unable to alloc intermediate buffer with size %zd", + size+1); + res = -ENOMEM; + goto out; + } + + res = scst_alloc_sysfs_work(scst_cm_mgmt, false, &work); + if (res != 0) + goto out; + + work->buf = i_buf; + + res = scst_sysfs_queue_wait_work(work); + if (res == 0) + res = size; + +out: + TRACE_EXIT_RES(res); + return res; +} + +static struct kobj_attribute scst_cm_mgmt_attr = + __ATTR(mgmt, S_IRUGO|S_IWUSR, scst_cm_mgmt_show, scst_cm_mgmt_store); + +static const struct attribute *scst_cm_attrs[] = { + &scst_cm_allow_not_conn_copy_attr.attr, + &scst_cm_mgmt_attr.attr, + NULL, +}; + +#endif /* #ifndef CONFIG_SCST_PROC */ + +static int scst_cm_get_initiator_port_transport_id(struct scst_tgt *tgt, + struct scst_session *scst_sess, uint8_t **transport_id) +{ + int res = 0; + uint8_t *tid = NULL; + + TRACE_ENTRY(); + + BUILD_BUG_ON((sizeof(SCST_CM_TID_ID)+3) > SCST_CM_TID_SIZE); + BUILD_BUG_ON(TID_COMMON_SIZE != SCST_CM_TID_SIZE); + + if (scst_sess == NULL) { + res = SCST_TRANSPORTID_PROTOCOLID_COPY_MGR; + goto out; + } + + tid = kzalloc(SCST_CM_TID_SIZE, GFP_KERNEL); + if (tid == NULL) { + PRINT_ERROR("Allocation of TransportID (size %d) failed", + SCST_CM_TID_SIZE); + res = -ENOMEM; + goto out; + } + + tid[0] = SCST_TRANSPORTID_PROTOCOLID_COPY_MGR; + memcpy(&tid[2], SCST_CM_TID_ID, sizeof(SCST_CM_TID_ID)); + + *transport_id = tid; + +out: + TRACE_EXIT_RES(res); + return res; +} + +static int scst_cm_release(struct scst_tgt *tgt) +{ + TRACE_ENTRY(); + TRACE_EXIT(); + return 0; +} + +static int scst_cm_xmit_response(struct scst_cmd *cmd) +{ + int res = SCST_TGT_RES_SUCCESS; + scst_i_finish_fn_t f = (void *) *((unsigned long long **)cmd->tgt_i_priv); + + TRACE_ENTRY(); + + /* + * Used for CM-only generated commands, i.e. commands generated without + * any external command, like INQUIRY. + */ + + f(cmd); + scst_tgt_cmd_done(cmd, SCST_CONTEXT_SAME); + + TRACE_EXIT_RES(res); + return res; +} + +static void scst_cm_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd) +{ + /* Nothing to do */ + return; +} + +static struct scst_tgt_template scst_cm_tgtt = { + .name = SCST_CM_NAME, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) + .sg_tablesize = SG_MAX_SINGLE_ALLOC, +#else + .sg_tablesize = 0xffff, +#endif +#ifndef CONFIG_SCST_PROC + .enabled_attr_not_needed = 1, +#endif + .dif_supported = 1, + .hw_dif_type1_supported = 1, + .hw_dif_type2_supported = 1, + .hw_dif_type3_supported = 1, + .release = scst_cm_release, + .xmit_response = scst_cm_xmit_response, + .task_mgmt_fn_done = scst_cm_task_mgmt_fn_done, + .get_initiator_port_transport_id = scst_cm_get_initiator_port_transport_id, +#ifndef CONFIG_SCST_PROC + .tgtt_attrs = scst_cm_attrs, +#endif +}; + +int __init scst_cm_init(void) +{ + int res = 0; + + TRACE_ENTRY(); + + spin_lock_init(&scst_cm_lock); + + res = scst_register_target_template(&scst_cm_tgtt); + if (res != 0) { + PRINT_ERROR("Unable to register copy manager template: %d", res); + goto out; + } + + scst_cm_tgt = scst_register_target(&scst_cm_tgtt, SCST_CM_TGT_NAME); + if (scst_cm_tgt == NULL) { + PRINT_ERROR("%s", "scst_register_target() failed"); + res = -EFAULT; + goto out_unreg_tgtt; + } + + scst_cm_sess = scst_register_session(scst_cm_tgt, false, + SCST_CM_SESS_NAME, NULL, NULL, NULL); + if (scst_cm_sess == NULL) { + PRINT_ERROR("%s", "scst_register_session() failed"); + res = -EFAULT; + goto out_unreg_tgt; + } + +out: + TRACE_EXIT_RES(res); + return res; + +out_unreg_tgt: + scst_unregister_target(scst_cm_tgt); + +out_unreg_tgtt: + scst_unregister_target_template(&scst_cm_tgtt); + goto out; +} + +void __exit scst_cm_exit(void) +{ + TRACE_ENTRY(); + + scst_unregister_session(scst_cm_sess, true, NULL); + scst_unregister_target(scst_cm_tgt); + scst_unregister_target_template(&scst_cm_tgtt); + + TRACE_EXIT(); + return; +} + +#endif /* #ifndef CONFIG_SCST_PROC */ diff --git a/scst/src/scst_debug.c b/scst/src/scst_debug.c index e470cce10..176e6dcad 100644 --- a/scst/src/scst_debug.c +++ b/scst/src/scst_debug.c @@ -232,6 +232,10 @@ const char *debug_transport_id_to_initiator_name(const uint8_t *transport_id) transport_id[8], transport_id[9], transport_id[10], transport_id[11]); break; + case SCST_TRANSPORTID_PROTOCOLID_COPY_MGR: + scnprintf(name_buf, SIZEOF_NAME_BUF, + "%s", &transport_id[2]); + break; default: scnprintf(name_buf, SIZEOF_NAME_BUF, "(Not known protocol ID %x)", transport_id[0] & 0x0f); diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index f883c4633..c447d2c26 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -172,6 +172,18 @@ const struct scst_opcode_descriptor scst_op_descr_inquiry = { }; EXPORT_SYMBOL(scst_op_descr_inquiry); +const struct scst_opcode_descriptor scst_op_descr_extended_copy = { + .od_opcode = EXTENDED_COPY, + .od_support = 3, /* supported as in the standard */ + .od_cdb_size = 16, + .od_nominal_timeout = SCST_DEFAULT_NOMINAL_TIMEOUT_SEC, + .od_recommended_timeout = SCST_GENERIC_DISK_REG_TIMEOUT/HZ, + .od_cdb_usage_bits = { EXTENDED_COPY, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0xFF, 0xFF, 0xFF, 0xFF, 0, + SCST_OD_DEFAULT_CONTROL_BYTE }, +}; +EXPORT_SYMBOL(scst_op_descr_extended_copy); + const struct scst_opcode_descriptor scst_op_descr_tur = { .od_opcode = TEST_UNIT_READY, .od_support = 3, /* supported as in the standard */ @@ -468,6 +480,8 @@ static int get_cdb_info_write_same16(struct scst_cmd *cmd, const struct scst_sdbops *sdbops); static int get_cdb_info_compare_and_write(struct scst_cmd *cmd, const struct scst_sdbops *sdbops); +static int get_cdb_info_ext_copy(struct scst_cmd *cmd, + const struct scst_sdbops *sdbops); static int get_cdb_info_apt(struct scst_cmd *cmd, const struct scst_sdbops *sdbops); static int get_cdb_info_min(struct scst_cmd *cmd, @@ -1256,16 +1270,18 @@ static const struct scst_sdbops scst_scsi_op_table[] = { .info_op_flags = SCST_WRITE_MEDIUM, .info_len_off = 10, .info_len_len = 4, .get_cdb_info = get_cdb_info_len_4}, - {.ops = 0x83, .devkey = "OOOOOOOOOOOOOOOO", + {.ops = 0x83, .devkey = "O ", /* implemented only for disks */ .info_op_name = "EXTENDED COPY", .info_data_direction = SCST_DATA_WRITE, - .info_op_flags = SCST_WRITE_MEDIUM, + .info_op_flags = SCST_WRITE_MEDIUM|SCST_CAN_GEN_3PARTY_COMMANDS| + SCST_LOCAL_CMD|SCST_DESCRIPTORS_BASED, .info_len_off = 10, .info_len_len = 4, - .get_cdb_info = get_cdb_info_len_4}, - {.ops = 0x84, .devkey = "OOOOOOOOOOOOOOOO", + .get_cdb_info = get_cdb_info_ext_copy}, + {.ops = 0x84, .devkey = "O ", /* implemented only for disks */ .info_op_name = "RECEIVE COPY RESULT", .info_data_direction = SCST_DATA_READ, - .info_op_flags = SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED, + .info_op_flags = SCST_FULLY_LOCAL_CMD|SCST_LOCAL_CMD| + SCST_WRITE_EXCL_ALLOWED|SCST_EXCL_ACCESS_ALLOWED, .info_len_off = 10, .info_len_len = 4, .get_cdb_info = get_cdb_info_len_4}, {.ops = 0x85, .devkey = "O O O ", @@ -6675,6 +6691,9 @@ struct scst_session *scst_alloc_session(struct scst_tgt *tgt, gfp_t gfp_mask, sess->tgt = tgt; INIT_LIST_HEAD(&sess->init_deferred_cmd_list); INIT_LIST_HEAD(&sess->init_deferred_mcmd_list); + INIT_LIST_HEAD(&sess->sess_cm_list_id_list); + INIT_DELAYED_WORK(&sess->sess_cm_list_id_cleanup_work, + (void (*)(struct work_struct *))sess_cm_list_id_cleanup_work_fn); #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20)) INIT_DELAYED_WORK(&sess->hw_pending_work, scst_hw_pending_work_fn); #else @@ -11215,6 +11234,20 @@ static int get_cdb_info_compare_and_write(struct scst_cmd *cmd, return scst_parse_wrprotect(cmd); } +static int get_cdb_info_ext_copy(struct scst_cmd *cmd, + const struct scst_sdbops *sdbops) +{ + if (unlikely(cmd->cdb[1] != 0)) { + PRINT_WARNING("Not supported %s service action 0x%x", + scst_get_opcode_name(cmd), cmd->cdb[1]); + scst_set_invalid_field_in_cdb(cmd, 1, + 0 | SCST_INVAL_FIELD_BIT_OFFS_VALID); + return 1; + } + + return get_cdb_info_len_4(cmd, sdbops); +} + /** * get_cdb_info_apt() - Parse ATA PASS-THROUGH CDB. * @@ -12781,7 +12814,7 @@ bool __scst_check_blocked_dev(struct scst_cmd *cmd) TRACE_ENTRY(); EXTRACHECKS_BUG_ON(cmd->unblock_dev); - EXTRACHECKS_BUG_ON(cmd->internal); + EXTRACHECKS_BUG_ON(cmd->internal && cmd->cdb[0] != EXTENDED_COPY); if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) goto out; @@ -13616,6 +13649,9 @@ int scst_parse_descriptors(struct scst_cmd *cmd) case UNMAP: res = scst_parse_unmap_descriptors(cmd); break; + case EXTENDED_COPY: + res = scst_cm_parse_descriptors(cmd); + break; default: sBUG(); } @@ -13632,6 +13668,9 @@ static void scst_free_descriptors(struct scst_cmd *cmd) case UNMAP: scst_free_unmap_descriptors(cmd); break; + case EXTENDED_COPY: + scst_cm_free_descriptors(cmd); + break; default: sBUG(); break; @@ -15112,6 +15151,12 @@ void scst_set_pre_exec_time(struct scst_cmd *cmd) TRACE_DBG("cmd %p: pre_exec_time %lld", cmd, cmd->pre_exec_time); } +void scst_set_exec_start(struct scst_cmd *cmd) +{ + cmd->exec_time_counting = true; + scst_set_cur_start(cmd); +} + void scst_set_exec_time(struct scst_cmd *cmd) { cmd->exec_time += scst_get_usec() - cmd->curr_start; diff --git a/scst/src/scst_main.c b/scst/src/scst_main.c index ebfb05803..67cbc797a 100644 --- a/scst/src/scst_main.c +++ b/scst/src/scst_main.c @@ -1469,6 +1469,10 @@ int scst_register_virtual_device(struct scst_dev_type *dev_handler, list_add_tail(&dev->dev_list_entry, &scst_dev_list); + res = scst_cm_on_dev_register(dev); + if (res != 0) + goto out_unreg; + mutex_unlock(&scst_mutex); scst_resume_activity(); @@ -1480,6 +1484,12 @@ out: TRACE_EXIT_RES(res); return res; +out_unreg: + dev->dev_unregistering = 1; + list_del(&dev->dev_list_entry); + scst_assign_dev_handler(dev, &scst_null_devtype); + goto out_pr_clear_dev; + #ifndef CONFIG_SCST_PROC out_lock_pr_clear_dev: mutex_lock(&scst_mutex); @@ -1532,6 +1542,8 @@ void scst_unregister_virtual_device(int id) dev->dev_unregistering = 1; + scst_cm_on_dev_unregister(dev); + list_del_init(&dev->dev_list_entry); scst_pr_clear_dev(dev); @@ -2659,6 +2671,15 @@ static int __init init_scst(void) goto out_thread_free; #endif + res = scst_cm_init(); + if (res != 0) +#ifdef CONFIG_SCST_PROC + goto out_proc_cleanup; +#else + goto out_thread_free; +#endif + + PRINT_INFO("SCST version %s loaded successfully (max mem for " "commands %dMB, per device %dMB)", SCST_VERSION_STRING, scst_max_cmd_mem, scst_max_dev_cmd_mem); @@ -2669,6 +2690,11 @@ out: TRACE_EXIT_RES(res); return res; +#ifdef CONFIG_SCST_PROC +out_proc_cleanup: + scst_proc_cleanup_module(); +#endif + out_thread_free: scst_stop_global_threads(); @@ -2751,6 +2777,8 @@ static void __exit exit_scst(void) /* ToDo: unregister_cpu_notifier() */ + scst_cm_exit(); + #ifdef CONFIG_SCST_PROC scst_proc_cleanup_module(); #endif diff --git a/scst/src/scst_priv.h b/scst/src/scst_priv.h index f6010f85f..f0606d677 100644 --- a/scst/src/scst_priv.h +++ b/scst/src/scst_priv.h @@ -713,6 +713,10 @@ void scst_gen_aen_or_ua(struct scst_tgt_dev *tgt_dev, void scst_block_dev(struct scst_device *dev); void scst_unblock_dev(struct scst_device *dev); +bool scst_do_check_blocked_dev(struct scst_cmd *cmd); +bool __scst_check_blocked_dev(struct scst_cmd *cmd); +void __scst_check_unblock_dev(struct scst_cmd *cmd); +void scst_check_unblock_dev(struct scst_cmd *cmd); int scst_ext_block_dev(struct scst_device *dev, bool sync, ext_blocker_done_fn_t done_fn, const uint8_t *priv, int priv_len); @@ -720,8 +724,6 @@ void scst_ext_unblock_dev(struct scst_device *dev, bool stpg); void __scst_ext_blocking_done(struct scst_device *dev); void scst_ext_blocking_done(struct scst_device *dev); -bool __scst_check_blocked_dev(struct scst_cmd *cmd); - /* * Increases global SCST ref counters which prevent from entering into suspended * activities stage, so protects from any global management operations. @@ -842,6 +844,66 @@ static inline bool scst_lba1_inside_lba2(int64_t lba1, return res; } +#ifndef CONFIG_SCST_PROC + +int scst_cm_on_dev_register(struct scst_device *dev); +void scst_cm_on_dev_unregister(struct scst_device *dev); + +int scst_cm_parse_descriptors(struct scst_cmd *cmd); +void scst_cm_free_descriptors(struct scst_cmd *cmd); + +int scst_cm_ext_copy_exec(struct scst_cmd *cmd); +int scst_cm_rcv_copy_res_exec(struct scst_cmd *cmd); + +void sess_cm_list_id_cleanup_work_fn(struct delayed_work *work); +void scst_cm_free_pending_list_ids(struct scst_session *sess); + +bool scst_cm_check_block_all_devs(struct scst_cmd *cmd); +void scst_cm_abort_ec_cmd(struct scst_cmd *ec_cmd); + +bool scst_cm_ec_cmd_overlap(struct scst_cmd *ec_cmd, struct scst_cmd *cmd); + +int scst_cm_init(void); +void scst_cm_exit(void); + +#else /* #ifndef CONFIG_SCST_PROC */ + +static inline int scst_cm_on_dev_register(struct scst_device *dev) { return 0; } +static inline void scst_cm_on_dev_unregister(struct scst_device *dev) {} + +static inline int scst_cm_parse_descriptors(struct scst_cmd *cmd) +{ + scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode)); + scst_set_cmd_abnormal_done_state(cmd); + return -1; +} +static inline void scst_cm_free_descriptors(struct scst_cmd *cmd) {} + +static inline int scst_cm_ext_copy_exec(struct scst_cmd *cmd) +{ + return SCST_EXEC_NOT_COMPLETED; +} +static inline int scst_cm_rcv_copy_res_exec(struct scst_cmd *cmd) +{ + return SCST_EXEC_NOT_COMPLETED; +} + +static inline void sess_cm_list_id_cleanup_work_fn(struct work_struct *work) {} +static inline void scst_cm_free_pending_list_ids(struct scst_session *sess) {} + +static inline bool scst_cm_check_block_all_devs(struct scst_cmd *cmd) { return false; } +static inline void scst_cm_abort_ec_cmd(struct scst_cmd *ec_cmd) {} + +static inline bool scst_cm_ec_cmd_overlap(struct scst_cmd *ec_cmd, struct scst_cmd *cmd) +{ + return false; +} + +static inline int scst_cm_init(void) { return 0; } +static inline void scst_cm_exit(void) {} + +#endif /* #ifndef CONFIG_SCST_PROC */ + #ifdef CONFIG_SCST_DEBUG_TM extern void tm_dbg_check_released_cmds(void); extern int tm_dbg_check_cmd(struct scst_cmd *cmd); @@ -901,6 +963,7 @@ void scst_set_alloc_buf_time(struct scst_cmd *cmd); void scst_set_restart_waiting_time(struct scst_cmd *cmd); void scst_set_rdy_to_xfer_time(struct scst_cmd *cmd); void scst_set_pre_exec_time(struct scst_cmd *cmd); +void scst_set_exec_start(struct scst_cmd *cmd); void scst_set_exec_time(struct scst_cmd *cmd); void scst_set_dev_done_time(struct scst_cmd *cmd); void scst_set_xmit_time(struct scst_cmd *cmd); @@ -915,6 +978,7 @@ static inline void scst_set_alloc_buf_time(struct scst_cmd *cmd) {} static inline void scst_set_restart_waiting_time(struct scst_cmd *cmd) {} static inline void scst_set_rdy_to_xfer_time(struct scst_cmd *cmd) {} static inline void scst_set_pre_exec_time(struct scst_cmd *cmd) {} +static inline void scst_set_exec_start(struct scst_cmd *cmd) {} static inline void scst_set_exec_time(struct scst_cmd *cmd) {} static inline void scst_set_dev_done_time(struct scst_cmd *cmd) {} static inline void scst_set_xmit_time(struct scst_cmd *cmd) {} diff --git a/scst/src/scst_targ.c b/scst/src/scst_targ.c index e2218d793..f6d8a01e7 100644 --- a/scst/src/scst_targ.c +++ b/scst/src/scst_targ.c @@ -162,6 +162,9 @@ static bool scst_cmd_overlap_cwr(struct scst_cmd *cwr_cmd, struct scst_cmd *cmd) res = scst_unmap_overlap(cmd, cwr_cmd->lba, cwr_cmd->data_len); break; + case EXTENDED_COPY: + res = scst_cm_ec_cmd_overlap(cmd, cwr_cmd); + break; default: res = false; break; @@ -430,12 +433,21 @@ static bool scst_check_blocked_dev(struct scst_cmd *cmd) TRACE_ENTRY(); - if (unlikely(cmd->internal)) { + if (unlikely((cmd->op_flags & SCST_CAN_GEN_3PARTY_COMMANDS) != 0)) { + EXTRACHECKS_BUG_ON(cmd->cdb[0] != EXTENDED_COPY); + res = scst_cm_check_block_all_devs(cmd); + goto out; + } + + if (unlikely(cmd->internal || cmd->bypass_blocking)) { /* * The original command can already block the device and must * hold reference to it, so internal command should always pass. */ - sBUG_ON(dev->on_dev_cmd_count == 0); + + /* Copy Manager can send internal INQUIRYs, so don't BUG on them */ + sBUG_ON((dev->on_dev_cmd_count == 0) && (cmd->cdb[0] != INQUIRY)); + res = false; goto out; } @@ -2087,7 +2099,17 @@ static int scst_tgt_pre_exec(struct scst_cmd *cmd) cmd->state = SCST_CMD_STATE_EXEC_CHECK_SN; - if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal)) + if (unlikely(cmd->internal)) { + if (cmd->dh_data_buf_alloced && cmd->tgt_i_data_buf_alloced && + (scst_cmd_get_data_direction(cmd) & SCST_DATA_WRITE)) { + TRACE_DBG("Internal WRITE cmd %p with DH alloced data", + cmd); + scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET); + } + goto out_descr; + } + + if (cmd->tgtt->pre_exec == NULL) goto out_descr; TRACE_DBG("Calling pre_exec(%p)", cmd); @@ -3161,7 +3183,7 @@ int __scst_check_local_events(struct scst_cmd *cmd, bool preempt_tests_only) TRACE_ENTRY(); - if (unlikely(cmd->internal)) { + if (unlikely(cmd->internal && !cmd->internal_check_local_events)) { if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) { TRACE_MGMT_DBG("ABORTED set, aborting internal " "cmd %p", cmd); @@ -3399,7 +3421,7 @@ static int scst_do_real_exec(struct scst_cmd *cmd) if (devt->exec) { TRACE_DBG("Calling dev handler %s exec(%p)", devt->name, cmd); - scst_set_cur_start(cmd); + scst_set_exec_start(cmd); res = devt->exec(cmd); TRACE_DBG("Dev handler %s exec() returned %d", devt->name, res); @@ -3425,7 +3447,7 @@ static int scst_do_real_exec(struct scst_cmd *cmd) goto out_error; } - scst_set_cur_start(cmd); + scst_set_exec_start(cmd); #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30) rc = scst_exec_req(scsi_dev, cmd->cdb, cmd->cdb_len, @@ -3519,6 +3541,8 @@ static scst_local_exec_fn scst_local_fns[256] = { [REPORT_LUNS] = scst_report_luns_local, [REQUEST_SENSE] = scst_request_sense_local, [COMPARE_AND_WRITE] = scst_cmp_wr_local, + [EXTENDED_COPY] = scst_cm_ext_copy_exec, + [RECEIVE_COPY_RESULTS] = scst_cm_rcv_copy_res_exec, [MAINTENANCE_IN] = scst_maintenance_in, }; @@ -5785,6 +5809,9 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, TRACE_ENTRY(); + /* Fantom EC commands must not leak here */ + sBUG_ON((cmd->cdb[0] == EXTENDED_COPY) && cmd->internal); + /* * Help Coverity recognize that mcmd != NULL if * call_dev_task_mgmt_fn_received == true. @@ -5834,6 +5861,9 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, */ smp_mb__after_set_bit(); + if (cmd->cdb[0] == EXTENDED_COPY) + scst_cm_abort_ec_cmd(cmd); + if (cmd->tgt_dev == NULL) { spin_lock_irqsave(&scst_init_lock, flags); scst_init_poll_cnt++; @@ -6911,6 +6941,16 @@ static int scst_mgmt_affected_cmds_done(struct scst_mgmt_cmd *mcmd) tgt_done: scst_call_task_mgmt_affected_cmds_done(mcmd); + switch (mcmd->fn) { + case SCST_LUN_RESET: + case SCST_TARGET_RESET: + case SCST_NEXUS_LOSS_SESS: + case SCST_NEXUS_LOSS: + case SCST_UNREG_SESS_TM: + scst_cm_free_pending_list_ids(sess); + break; + } + res = scst_set_mcmd_next_state(mcmd); TRACE_EXIT_RES(res); diff --git a/usr/fileio/Makefile b/usr/fileio/Makefile index 4789564ef..b91948369 100644 --- a/usr/fileio/Makefile +++ b/usr/fileio/Makefile @@ -49,6 +49,7 @@ CFLAGS += $(LOCAL_CFLAGS) #CFLAGS += -DDEBUG_TM_IGNORE #CFLAGS += -DDEBUG_TM_IGNORE -DDEBUG_TM_FN_IGNORE #CFLAGS += -DDEBUG_TM_IGNORE_ALL +CFLAGS += -DDEBUG_EXT_COPY_REMAP all: $(PROGS) diff --git a/usr/fileio/common.c b/usr/fileio/common.c index 41400c44e..8c7938b47 100644 --- a/usr/fileio/common.c +++ b/usr/fileio/common.c @@ -69,11 +69,31 @@ static int open_dev_fd(struct vdisk_dev *dev) return res; } -static inline void set_cmd_error_status(struct scst_user_scsi_cmd_reply_exec *reply, +static void set_resp_data_len(struct vdisk_cmd *vcmd, int32_t resp_data_len) +{ + struct scst_user_scsi_cmd_reply_exec *reply = &vcmd->reply->exec_reply; + + TRACE_ENTRY(); + + if (vcmd->may_need_to_free_pbuf && (resp_data_len == 0)) { + struct scst_user_scsi_cmd_exec *cmd = &vcmd->cmd->exec_cmd; + free((void *)(unsigned long)cmd->pbuf); + cmd->pbuf = 0; + reply->pbuf = 0; + } + + reply->resp_data_len = resp_data_len; + + TRACE_EXIT(); + return; +} + +static inline void set_cmd_error_status(struct vdisk_cmd *vcmd, int status) { + struct scst_user_scsi_cmd_reply_exec *reply = &vcmd->reply->exec_reply; reply->status = status; - reply->resp_data_len = 0; + set_resp_data_len(vcmd, 0); return; } @@ -108,7 +128,7 @@ void set_cmd_error(struct vdisk_cmd *vcmd, int key, int asc, int ascq) EXTRACHECKS_BUG_ON(vcmd->cmd->subcode != SCST_USER_EXEC); - set_cmd_error_status(reply, SAM_STAT_CHECK_CONDITION); + set_cmd_error_status(vcmd, SAM_STAT_CHECK_CONDITION); reply->sense_len = set_sense(vcmd->sense, sizeof(vcmd->sense), key, asc, ascq); reply->psense_buffer = (unsigned long)vcmd->sense; @@ -117,11 +137,11 @@ void set_cmd_error(struct vdisk_cmd *vcmd, int key, int asc, int ascq) return; } -void set_busy(struct scst_user_scsi_cmd_reply_exec *reply) +void set_busy(struct vdisk_cmd *vcmd) { TRACE_ENTRY(); - set_cmd_error_status(reply, SAM_STAT_TASK_SET_FULL); + set_cmd_error_status(vcmd, SAM_STAT_TASK_SET_FULL); TRACE_MGMT_DBG("%s", "Sending QUEUE_FULL status"); TRACE_EXIT(); @@ -212,6 +232,9 @@ static int do_exec(struct vdisk_cmd *vcmd) TRACE_ENTRY(); + /* Must be reinitialized each time to avoid crash on stale value */ + vcmd->may_need_to_free_pbuf = 0; + switch(cmd->queue_type) { case SCST_CMD_QUEUE_ORDERED: TRACE(TRACE_ORDER, "ORDERED cmd_h %d", vcmd->cmd->cmd_h); @@ -251,6 +274,7 @@ static int do_exec(struct vdisk_cmd *vcmd) else #endif cmd->pbuf = (unsigned long)dev->alloc_fn(cmd->alloc_len); + vcmd->may_need_to_free_pbuf = 1; TRACE_MEM("Buf %"PRIx64" alloced, len %d", cmd->pbuf, cmd->alloc_len); reply->pbuf = cmd->pbuf; @@ -258,7 +282,7 @@ static int do_exec(struct vdisk_cmd *vcmd) TRACE(TRACE_OUT_OF_MEM, "Unable to allocate buffer " "(len %d)", cmd->alloc_len); #ifndef DEBUG_NOMEM - set_busy(reply); + set_busy(vcmd); #endif goto out; } @@ -524,6 +548,49 @@ static int do_on_free_cmd(struct vdisk_cmd *vcmd) return res; } +#ifdef DEBUG_EXT_COPY_REMAP +static int do_ext_copy_remap(struct vdisk_cmd *vcmd) +{ + struct scst_user_get_cmd *cmd = vcmd->cmd; + struct scst_user_ext_copy_remap *rcmd = &cmd->remap_cmd; + struct scst_user_reply_cmd *reply = vcmd->reply; + struct scst_user_ext_copy_reply_remap *rreply = &reply->remap_reply; + static struct scst_user_ext_copy_data_descr d[1]; + int res = 0; + + TRACE_ENTRY(); + + TRACE_DBG("Ext copy remap cmd %d", cmd->cmd_h); + + memset(reply, 0, sizeof(*reply)); + reply->cmd_h = cmd->cmd_h; + reply->subcode = cmd->subcode; + + memset(rreply, 0, sizeof(*rreply)); + + /* It's only a debug code, so it's OK to use static descr */ + + memset(d, 0, sizeof(d)); + + d[0].data_len = rcmd->data_descr.data_len; + d[0].src_lba = rcmd->data_descr.src_lba; + d[0].dst_lba = rcmd->data_descr.dst_lba; + +#if 1 + rreply->remap_descriptors = (unsigned long)d; + rreply->remap_descriptors_len = sizeof(d); +#else + rreply->status = SAM_STAT_CHECK_CONDITION; + rreply->sense_len = set_sense(vcmd->sense, sizeof(vcmd->sense), + SCST_LOAD_SENSE(scst_sense_data_protect)); + rreply->psense_buffer = (unsigned long)vcmd->sense; +#endif + + TRACE_EXIT_RES(res); + return res; +} +#endif + static int do_tm(struct vdisk_cmd *vcmd, int done) { struct scst_user_get_cmd *cmd = vcmd->cmd; @@ -668,6 +735,12 @@ static int process_cmd(struct vdisk_cmd *vcmd) res = do_on_free_cmd(vcmd); break; +#ifdef DEBUG_EXT_COPY_REMAP + case SCST_USER_EXT_COPY_REMAP: + res = do_ext_copy_remap(vcmd); + break; +#endif + case SCST_USER_TASK_MGMT_RECEIVED: res = do_tm(vcmd, 0); break; @@ -704,7 +777,14 @@ void *main_loop(void *arg) struct vdisk_dev *dev = (struct vdisk_dev *)arg; struct scst_user_get_cmd cmd; struct scst_user_reply_cmd reply; - struct vdisk_cmd vcmd = { -1, &cmd, dev, &reply, {0}}; + struct vdisk_cmd vcmd = { + .fd = -1, + .cmd = &cmd, + .dev = dev, + .may_need_to_free_pbuf = 0, + .reply = &reply, + .sense = {0} + }; int scst_usr_fd = dev->scst_usr_fd; struct pollfd pl; #define MULTI_CMDS_CNT 2 @@ -746,6 +826,7 @@ void *main_loop(void *arg) } } #endif + if (use_multi) { TRACE_DBG("preplies %p (first: %p), replies_cnt %d, " "replies_done %d, cmds_cnt %d", (void *)multi.multi_cmd.preplies, @@ -906,11 +987,13 @@ static void exec_inquiry(struct vdisk_cmd *vcmd) uint64_t dev_id_num = gen_dev_id_num(dev); if (0 == cmd->cdb[2]) { /* supported vital product data pages */ - buf[3] = 3; + buf[3] = 5; buf[4] = 0x0; /* this page */ buf[5] = 0x80; /* unit serial number */ buf[6] = 0x83; /* device identification */ - resp_len = buf[3] + 4; + buf[7] = 0xB0; /* block limits */ + buf[8] = 0xB1; /* block device characteristics */ + resp_len = buf[3] + 6; } else if (0x80 == cmd->cdb[2]) { /* unit serial number */ int usn_len = strlen(dev->usn); buf[1] = 0x80; @@ -957,6 +1040,55 @@ static void exec_inquiry(struct vdisk_cmd *vcmd) buf[2] = (resp_len >> 8) & 0xFF; buf[3] = resp_len & 0xFF; resp_len += 4; + } else if (0xB0 == cmd->cdb[2]) { + /* Block Limits */ + int max_transfer; + buf[1] = 0xB0; + buf[3] = 0x3C; + buf[5] = 0xFF; /* No MAXIMUM COMPARE AND WRITE LENGTH limit */ + /* Optimal transfer granuality is PAGE_SIZE */ + max_transfer = max((int)(4096/dev->block_size), (int)1); + buf[6] = (max_transfer >> 8) & 0xff; + buf[7] = max_transfer & 0xff; + /* + * Max transfer len is min of sg limit and 8M, but we + * don't have access to them here, so let's use 1M. + */ + max_transfer = 1*1024*1024; + buf[8] = (max_transfer >> 24) & 0xff; + buf[9] = (max_transfer >> 16) & 0xff; + buf[10] = (max_transfer >> 8) & 0xff; + buf[11] = max_transfer & 0xff; + /* + * Let's have optimal transfer len 512KB. Better to not + * set it at all, because we don't have such limit, + * but some initiators may not understand that (?). + * From other side, too big transfers are not optimal, + * because SGV cache supports only <4M buffers. + */ + max_transfer = min((int)max_transfer, (int)(512*1024 / dev->block_size)); + buf[12] = (max_transfer >> 24) & 0xff; + buf[13] = (max_transfer >> 16) & 0xff; + buf[14] = (max_transfer >> 8) & 0xff; + buf[15] = max_transfer & 0xff; + resp_len = buf[3] + 4; + } else if (0xB1 == cmd->cdb[2]) { + int r; + /* Block Device Characteristics */ + buf[1] = 0xB1; + buf[3] = 0x3C; +#if 0 + if (virt_dev->rotational) { +#endif + /* 15K RPM */ + r = 0x3A98; +#if 0 + } else + r = 1; +#endif + buf[4] = (r >> 8) & 0xff; + buf[5] = r & 0xff; + resp_len = buf[3] + 4; } else { TRACE_DBG("INQUIRY: Unsupported EVPD page %x", cmd->cdb[2]); @@ -1000,7 +1132,7 @@ static void exec_inquiry(struct vdisk_cmd *vcmd) memcpy(address, buf, length); if (length < reply->resp_data_len) - reply->resp_data_len = length; + set_resp_data_len(vcmd, length); out: TRACE_EXIT(); @@ -1024,7 +1156,7 @@ static void exec_request_sense(struct vdisk_cmd *vcmd) memcpy(address, b, length); if (length < reply->resp_data_len) - reply->resp_data_len = length; + set_resp_data_len(vcmd, length); TRACE_EXIT(); return; @@ -1265,7 +1397,7 @@ static void exec_mode_sense(struct vdisk_cmd *vcmd) memcpy(address, buf, offset); if (offset < reply->resp_data_len) - reply->resp_data_len = offset; + set_resp_data_len(vcmd, offset); out: TRACE_EXIT(); @@ -1396,7 +1528,7 @@ static void exec_read_capacity(struct vdisk_cmd *vcmd) memcpy(address, buffer, length); if (length < reply->resp_data_len) - reply->resp_data_len = length; + set_resp_data_len(vcmd, length); TRACE_EXIT(); return; @@ -1455,7 +1587,7 @@ static void exec_read_capacity16(struct vdisk_cmd *vcmd) memcpy(address, buffer, length); if (length < reply->resp_data_len) - reply->resp_data_len = length; + set_resp_data_len(vcmd, length); TRACE_EXIT(); return; @@ -1534,7 +1666,7 @@ static void exec_read_toc(struct vdisk_cmd *vcmd) memcpy(address, buffer, off); if (off < reply->resp_data_len) - reply->resp_data_len = off; + set_resp_data_len(vcmd, off); out: TRACE_EXIT(); @@ -1577,7 +1709,6 @@ static void exec_read(struct vdisk_cmd *vcmd, loff_t loff) { struct vdisk_dev *dev = vcmd->dev; struct scst_user_scsi_cmd_exec *cmd = &vcmd->cmd->exec_cmd; - struct scst_user_scsi_cmd_reply_exec *reply = &vcmd->reply->exec_reply; int length = cmd->bufflen; uint8_t *address = (uint8_t*)(unsigned long)cmd->pbuf; int fd = vcmd->fd; @@ -1606,7 +1737,7 @@ static void exec_read(struct vdisk_cmd *vcmd, loff_t loff) PRINT_ERROR("read() returned %"PRId64" from %d (errno %d)", (uint64_t)err, length, errno); if (err == -EAGAIN) - set_busy(reply); + set_busy(vcmd); else { set_cmd_error(vcmd, SCST_LOAD_SENSE(scst_sense_read_error)); @@ -1614,7 +1745,7 @@ static void exec_read(struct vdisk_cmd *vcmd, loff_t loff) goto out; } - reply->resp_data_len = cmd->bufflen; + set_resp_data_len(vcmd, cmd->bufflen); out: TRACE_EXIT(); @@ -1625,7 +1756,6 @@ static void exec_write(struct vdisk_cmd *vcmd, loff_t loff) { struct vdisk_dev *dev = vcmd->dev; struct scst_user_scsi_cmd_exec *cmd = &vcmd->cmd->exec_cmd; - struct scst_user_scsi_cmd_reply_exec *reply = &vcmd->reply->exec_reply; loff_t err; int length = cmd->bufflen; uint8_t *address = (uint8_t*)(unsigned long)cmd->pbuf; @@ -1658,7 +1788,7 @@ restart: PRINT_ERROR("write() returned %"PRId64" from %d (errno %d, " "cmd_h %x)", err, length, errno, vcmd->cmd->cmd_h); if (err == -EAGAIN) - set_busy(reply); + set_busy(vcmd); else { set_cmd_error(vcmd, SCST_LOAD_SENSE(scst_sense_write_error)); @@ -1687,7 +1817,6 @@ static void exec_verify(struct vdisk_cmd *vcmd, loff_t loff) { struct vdisk_dev *dev = vcmd->dev; struct scst_user_scsi_cmd_exec *cmd = &vcmd->cmd->exec_cmd; - struct scst_user_scsi_cmd_reply_exec *reply = &vcmd->reply->exec_reply; loff_t err; int64_t length = cmd->bufflen; uint8_t *address = (uint8_t *)(unsigned long)cmd->pbuf; @@ -1740,7 +1869,7 @@ static void exec_verify(struct vdisk_cmd *vcmd, loff_t loff) PRINT_ERROR("read() returned %"PRId64" from %"PRId64" " "(errno %d)", (uint64_t)err, len_mem, errno); if (err == -EAGAIN) - set_busy(reply); + set_busy(vcmd); else { set_cmd_error(vcmd, SCST_LOAD_SENSE(scst_sense_read_error)); diff --git a/usr/fileio/common.h b/usr/fileio/common.h index 8fb1f14aa..7ef29bb09 100644 --- a/usr/fileio/common.h +++ b/usr/fileio/common.h @@ -97,6 +97,7 @@ struct vdisk_cmd int fd; struct scst_user_get_cmd *cmd; struct vdisk_dev *dev; + unsigned int may_need_to_free_pbuf:1; struct scst_user_reply_cmd *reply; uint8_t sense[SCST_SENSE_BUFFERSIZE]; }; diff --git a/usr/fileio/fileio.c b/usr/fileio/fileio.c index c9ca1aaf1..ec19a1980 100644 --- a/usr/fileio/fileio.c +++ b/usr/fileio/fileio.c @@ -394,6 +394,9 @@ int start(int argc, char **argv) desc.opt.queue_alg = SCST_QUEUE_ALG_1_UNRESTRICTED_REORDER; desc.opt.qerr = SCST_QERR_0_ALL_RESUME; desc.opt.d_sense = SCST_D_SENSE_0_FIXED_SENSE; +#ifdef DEBUG_EXT_COPY_REMAP + desc.opt.ext_copy_remap_supported = 1; +#endif res = ioctl(devs[i].scst_usr_fd, SCST_USER_REGISTER_DEVICE, &desc); if (res != 0) {