From 8f289d5f8ee19fcae13f26cd0e93d06300aee70e Mon Sep 17 00:00:00 2001 From: Vladislav Bolkhovitin Date: Tue, 1 May 2012 22:25:37 +0000 Subject: [PATCH] WRITE SAME implemented git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@4262 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- fcst/ft_cmd.c | 6 +- scst/include/scst.h | 70 ++--- scst/src/dev_handlers/scst_vdisk.c | 70 ++++- scst/src/scst_lib.c | 416 ++++++++++++++++++++++++++--- scst/src/scst_targ.c | 83 +++--- 5 files changed, 546 insertions(+), 99 deletions(-) diff --git a/fcst/ft_cmd.c b/fcst/ft_cmd.c index 86da44c09..bd3938a7e 100644 --- a/fcst/ft_cmd.c +++ b/fcst/ft_cmd.c @@ -63,7 +63,7 @@ void ft_cmd_dump(struct scst_cmd *cmd, const char *caller) cmd->bufflen, cmd->out_bufflen); printk(KERN_INFO "%s sg_cnt reg %d in %d tgt %d tgt_in %d\n", prefix, cmd->sg_cnt, cmd->out_sg_cnt, - cmd->tgt_sg_cnt, cmd->tgt_out_sg_cnt); + cmd->tgt_i_sg_cnt, cmd->tgt_out_sg_cnt); buf[0] = '\0'; if (cmd->sent_for_exec) @@ -88,8 +88,8 @@ void ft_cmd_dump(struct scst_cmd *cmd, const char *caller) ft_cmd_flag(buf, sizeof(buf), "hw_pend"); if (cmd->tgt_need_alloc_data_buf) ft_cmd_flag(buf, sizeof(buf), "tgt_need_alloc"); - if (cmd->tgt_data_buf_alloced) - ft_cmd_flag(buf, sizeof(buf), "tgt_alloced"); + if (cmd->tgt_i_data_buf_alloced) + ft_cmd_flag(buf, sizeof(buf), "tgt_i_alloced"); if (cmd->dh_data_buf_alloced) ft_cmd_flag(buf, sizeof(buf), "dh_alloced"); if (cmd->expected_values_set) diff --git a/scst/include/scst.h b/scst/include/scst.h index 4e10e2c6c..55faaab5a 100644 --- a/scst/include/scst.h +++ b/scst/include/scst.h @@ -748,7 +748,7 @@ struct scst_tgt_template { * Shall return 0 in case of success or < 0 (preferably -ENOMEM) * in case of error, or > 0 if the regular SCST allocation should be * done. In case of returning successfully, - * scst_cmd->tgt_data_buf_alloced will be set by SCST. + * scst_cmd->tgt_i_data_buf_alloced will be set by SCST. * * It is possible that both target driver and dev handler request own * memory allocation. In this case, data will be memcpy() between @@ -1596,7 +1596,7 @@ struct scst_session { struct scst_tgt *tgt; /* corresponding target */ /* Used for storage of target driver private stuff */ - void *tgt_priv; + void *sess_tgt_priv; /* session's async flags */ unsigned long sess_aflags; @@ -1861,10 +1861,10 @@ struct scst_cmd { unsigned int tgt_need_alloc_data_buf:1; /* - * Set by SCST if the custom data buffer allocation by the target driver - * succeeded. + * Set by SCST if the custom data buffer allocated by the target driver + * or, for internal commands, by SCST core . */ - unsigned int tgt_data_buf_alloced:1; + unsigned int tgt_i_data_buf_alloced:1; /* Set if custom data buffer allocated by dev handler */ unsigned int dh_data_buf_alloced:1; @@ -2053,16 +2053,16 @@ struct scst_cmd { int out_sg_cnt; /* WRITE SG segments count */ /* - * Used if both target driver and dev handler request own memory - * allocation. In other cases, both are equal to sg and sg_cnt - * correspondingly. + * Used if both target driver or SCST core for internal commands and + * dev handler request own memory allocation. In other cases, both + * are equal to sg and sg_cnt correspondingly. * * If target driver requests own memory allocations, it MUST use * functions scst_cmd_get_tgt_sg*() to get sg and sg_cnt! Otherwise, * it may use functions scst_cmd_get_sg*(). */ - struct scatterlist *tgt_sg; - int tgt_sg_cnt; + struct scatterlist *tgt_i_sg; + int tgt_i_sg_cnt; struct scatterlist *tgt_out_sg; /* bidirectional */ int tgt_out_sg_cnt; /* bidirectional */ @@ -2082,8 +2082,8 @@ struct scst_cmd { /* Start time when cmd was sent to rdy_to_xfer() or xmit_response() */ unsigned long hw_pending_start; - /* Used for storage of target driver private stuff */ - void *tgt_priv; + /* Used for storage of target driver or internal commands private stuff */ + void *tgt_i_priv; /* Used for storage of dev handler private stuff */ void *dh_priv; @@ -2108,8 +2108,6 @@ struct scst_cmd { /* Counter of the corresponding SCST_PR_ABORT_ALL TM commands */ struct scst_pr_abort_all_pending_mgmt_cmds_counter *pr_abort_counter; - struct scst_cmd *orig_cmd; /* Used to issue REQUEST SENSE */ - #ifdef CONFIG_SCST_MEASURE_LATENCY /* * Must be the last to allow to work with drivers who don't know @@ -2979,13 +2977,13 @@ void scst_update_hw_pending_start(struct scst_cmd *cmd); */ static inline void *scst_sess_get_tgt_priv(struct scst_session *sess) { - return sess->tgt_priv; + return sess->sess_tgt_priv; } static inline void scst_sess_set_tgt_priv(struct scst_session *sess, void *val) { - sess->tgt_priv = val; + sess->sess_tgt_priv = val; } uint16_t scst_lookup_tg_id(struct scst_device *dev, struct scst_tgt *tgt); @@ -3192,25 +3190,34 @@ static inline unsigned int scst_cmd_get_out_bufflen(struct scst_cmd *cmd) return cmd->out_bufflen; } -/* Returns pointer to cmd's target's SG data buffer */ +/* + * Returns pointer to cmd's target's SG data buffer. Since it's for target + * drivers, the "_i_" part is omitted. + */ static inline struct scatterlist *scst_cmd_get_tgt_sg(struct scst_cmd *cmd) { - return cmd->tgt_sg; + return cmd->tgt_i_sg; } -/* Returns cmd's target's sg_cnt */ +/* + * Returns cmd's target's sg_cnt. Since it's for target + * drivers, the "_i_" part is omitted. + */ static inline int scst_cmd_get_tgt_sg_cnt(struct scst_cmd *cmd) { - return cmd->tgt_sg_cnt; + return cmd->tgt_i_sg_cnt; } -/* Sets cmd's target's SG data buffer */ +/* + * Sets cmd's target's SG data buffer. Since it's for target + * drivers, the "_i_" part is omitted. + */ static inline void scst_cmd_set_tgt_sg(struct scst_cmd *cmd, struct scatterlist *sg, int sg_cnt) { - cmd->tgt_sg = sg; - cmd->tgt_sg_cnt = sg_cnt; - cmd->tgt_data_buf_alloced = 1; + cmd->tgt_i_sg = sg; + cmd->tgt_i_sg_cnt = sg_cnt; + cmd->tgt_i_data_buf_alloced = 1; } /* Returns pointer to cmd's target's OUT SG data buffer */ @@ -3229,7 +3236,7 @@ static inline int scst_cmd_get_tgt_out_sg_cnt(struct scst_cmd *cmd) static inline void scst_cmd_set_tgt_out_sg(struct scst_cmd *cmd, struct scatterlist *sg, int sg_cnt) { - WARN_ON(!cmd->tgt_data_buf_alloced); + WARN_ON(!cmd->tgt_i_data_buf_alloced); cmd->tgt_out_sg = sg; cmd->tgt_out_sg_cnt = sg_cnt; @@ -3340,12 +3347,12 @@ static inline void scst_cmd_set_tag(struct scst_cmd *cmd, uint64_t tag) */ static inline void *scst_cmd_get_tgt_priv(struct scst_cmd *cmd) { - return cmd->tgt_priv; + return cmd->tgt_i_priv; } static inline void scst_cmd_set_tgt_priv(struct scst_cmd *cmd, void *val) { - cmd->tgt_priv = val; + cmd->tgt_i_priv = val; } /* @@ -3362,16 +3369,17 @@ static inline void scst_cmd_set_tgt_need_alloc_data_buf(struct scst_cmd *cmd) } /* - * Get/Set functions for tgt_data_buf_alloced flag + * Get/Set functions for tgt_i_data_buf_alloced flag. Since they are for target + * drivers, the "_i_" part is omitted. */ static inline int scst_cmd_get_tgt_data_buff_alloced(struct scst_cmd *cmd) { - return cmd->tgt_data_buf_alloced; + return cmd->tgt_i_data_buf_alloced; } static inline void scst_cmd_set_tgt_data_buff_alloced(struct scst_cmd *cmd) { - cmd->tgt_data_buf_alloced = 1; + cmd->tgt_i_data_buf_alloced = 1; } /* @@ -4383,4 +4391,6 @@ int scst_scsi_exec_async(struct scst_cmd *cmd, void *data, void (*done)(void *data, char *sense, int result, int resid)); #endif +void scst_write_same(struct scst_cmd *cmd); + #endif /* __SCST_H */ diff --git a/scst/src/dev_handlers/scst_vdisk.c b/scst/src/dev_handlers/scst_vdisk.c index d05eb024f..afd67648d 100644 --- a/scst/src/dev_handlers/scst_vdisk.c +++ b/scst/src/dev_handlers/scst_vdisk.c @@ -258,6 +258,7 @@ static enum compl_status_e vdisk_exec_log(struct vdisk_cmd_params *p); static enum compl_status_e vdisk_exec_read_toc(struct vdisk_cmd_params *p); static enum compl_status_e vdisk_exec_prevent_allow_medium_removal(struct vdisk_cmd_params *p); static enum compl_status_e vdisk_exec_unmap(struct vdisk_cmd_params *p); +static enum compl_status_e vdisk_exec_write_same(struct vdisk_cmd_params *p); static int vdisk_fsync(struct vdisk_cmd_params *p, loff_t loff, loff_t len, struct scst_device *dev, gfp_t gfp_flags, struct scst_cmd *cmd); @@ -1083,6 +1084,8 @@ static enum compl_status_e vdisk_invalid_opcode(struct vdisk_cmd_params *p) [READ_CAPACITY] = vdisk_exec_read_capacity, \ [SERVICE_ACTION_IN] = vdisk_exec_srv_action_in, \ [UNMAP] = vdisk_exec_unmap, \ + [WRITE_SAME] = vdisk_exec_write_same, \ + [WRITE_SAME_16] = vdisk_exec_write_same, \ [MAINTENANCE_IN] = vdisk_exec_maintenance_in, \ [SEND_DIAGNOSTIC] = vdisk_exec_send_diagnostic, @@ -1626,7 +1629,7 @@ static int fileio_alloc_data_buf(struct scst_cmd *cmd) * itself or the command is a write or bidi command, don't use zero * copy. */ - if (cmd->tgt_data_buf_alloced || + if (cmd->tgt_i_data_buf_alloced || (cmd->data_direction & SCST_DATA_READ) == 0) { p->use_zero_copy = false; } @@ -1946,6 +1949,66 @@ out: return res; } +static void vdisk_exec_write_same_unmap(struct vdisk_cmd_params *p) +{ + int rc; + struct scst_cmd *cmd = p->cmd; + struct scst_device *dev = cmd->dev; + struct scst_vdisk_dev *virt_dev = dev->dh_priv; + + TRACE_ENTRY(); + + if (unlikely(!virt_dev->thin_provisioned)) { + TRACE_DBG("%s", "Device not thin provisioned"); + scst_set_cmd_error(cmd, + SCST_LOAD_SENSE(scst_sense_invalid_opcode)); + goto out; + } + + rc = vdisk_unmap_range(cmd, virt_dev, cmd->lba, + cmd->data_len >> dev->block_shift); + if (rc != 0) + goto out; + +out: + TRACE_EXIT(); + return; +} + +static enum compl_status_e vdisk_exec_write_same(struct vdisk_cmd_params *p) +{ + struct scst_cmd *cmd = p->cmd; + enum compl_status_e res = CMD_SUCCEEDED; + + TRACE_ENTRY(); + + if (unlikely(cmd->cdb[1] & 1)) { + TRACE_DBG("%s", "ANCHOR not supported"); + scst_set_cmd_error(cmd, + SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb)); + goto out; + } + + if (unlikely(cmd->cdb[1] & 0xE0)) { + TRACE_DBG("%s", "WRPROTECT not supported"); + scst_set_cmd_error(cmd, + SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb)); + goto out; + } + + if (cmd->cdb[1] & 0x8) { + vdisk_exec_write_same_unmap(p); + goto out; + } + + scst_write_same(cmd); + res = RUNNING_ASYNC; + +out: + TRACE_EXIT_RES(res); + return res; +} + static enum compl_status_e vdisk_exec_unmap(struct vdisk_cmd_params *p) { struct scst_cmd *cmd = p->cmd; @@ -1957,15 +2020,14 @@ static enum compl_status_e vdisk_exec_unmap(struct vdisk_cmd_params *p) TRACE_ENTRY(); if (unlikely(!virt_dev->thin_provisioned)) { - TRACE_DBG("%s", "Invalid opcode UNMAP"); + TRACE_DBG("%s", "Device not thin provisioned"); scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_opcode)); goto out; } if (unlikely(cmd->cdb[1] & 1)) { - /* ANCHOR not supported */ - TRACE_DBG("%s", "Invalid ANCHOR field"); + TRACE_DBG("%s", "ANCHOR not supported"); scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb)); goto out; diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index 404863a95..3963ab76e 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -67,6 +67,8 @@ static int strncasecmp(const char *s1, const char *s2, size_t n) } #endif +static void scst_destroy_put_cmd(struct scst_cmd *cmd); + struct scst_sdbops; static int get_cdb_info_len_10(struct scst_cmd *cmd, @@ -125,6 +127,10 @@ static int get_cdb_info_lba_8_len_4(struct scst_cmd *cmd, const struct scst_sdbops *sdbops); static int get_cdb_info_lba_8_none(struct scst_cmd *cmd, const struct scst_sdbops *sdbops); +static int get_cdb_info_write_same10(struct scst_cmd *cmd, + const struct scst_sdbops *sdbops); +static int get_cdb_info_write_same16(struct scst_cmd *cmd, + const struct scst_sdbops *sdbops); /* +=====================================-============-======- @@ -657,13 +663,13 @@ static const struct scst_sdbops scst_scsi_op_table[] = { .info_op_flags = SCST_SMALL_TIMEOUT, .info_len_off = 8, .info_len_len = 1, .get_cdb_info = get_cdb_info_len_1}, - {.ops = 0x41, .devkey = "O O ", + {.ops = 0x41, .devkey = "O ", .info_op_name = "WRITE SAME(10)", .info_data_direction = SCST_DATA_WRITE, .info_op_flags = SCST_TRANSFER_LEN_TYPE_FIXED|SCST_WRITE_MEDIUM, .info_lba_off = 2, .info_lba_len = 4, .info_len_off = 7, .info_len_len = 2, - .get_cdb_info = get_cdb_info_lba_4_len_2}, + .get_cdb_info = get_cdb_info_write_same10}, {.ops = 0x42, .devkey = " O ", .info_op_name = "READ SUB-CHANNEL", .info_data_direction = SCST_DATA_READ, @@ -975,13 +981,13 @@ static const struct scst_sdbops scst_scsi_op_table[] = { .info_data_direction = SCST_DATA_NONE, .info_op_flags = SCST_LONG_TIMEOUT|SCST_WRITE_EXCL_ALLOWED, .get_cdb_info = get_cdb_info_none}, - {.ops = 0x93, .devkey = "O O ", + {.ops = 0x93, .devkey = "O ", .info_op_name = "WRITE SAME(16)", .info_data_direction = SCST_DATA_WRITE, .info_op_flags = SCST_TRANSFER_LEN_TYPE_FIXED|SCST_WRITE_MEDIUM, .info_lba_off = 2, .info_lba_len = 8, .info_len_off = 10, .info_len_len = 4, - .get_cdb_info = get_cdb_info_lba_8_len_4}, + .get_cdb_info = get_cdb_info_write_same16}, {.ops = 0x93, .devkey = " M ", .info_op_name = "ERASE(16)", .info_data_direction = SCST_DATA_NONE, @@ -1419,9 +1425,9 @@ static int scst_set_lun_not_supported_request_sense(struct scst_cmd *cmd, * callback, it is responsible to copy the sense to its buffer * in xmit_response(). */ - if (cmd->tgt_data_buf_alloced && (cmd->tgt_sg != NULL)) { - cmd->sg = cmd->tgt_sg; - cmd->sg_cnt = cmd->tgt_sg_cnt; + if (cmd->tgt_i_data_buf_alloced && (cmd->tgt_i_sg != NULL)) { + cmd->sg = cmd->tgt_i_sg; + cmd->sg_cnt = cmd->tgt_i_sg_cnt; TRACE_MEM("Tgt sg used for sense for cmd %p", cmd); goto go; } @@ -1486,9 +1492,9 @@ static int scst_set_lun_not_supported_inquiry(struct scst_cmd *cmd) * callback, it is responsible to copy the sense to its buffer * in xmit_response(). */ - if (cmd->tgt_data_buf_alloced && (cmd->tgt_sg != NULL)) { - cmd->sg = cmd->tgt_sg; - cmd->sg_cnt = cmd->tgt_sg_cnt; + if (cmd->tgt_i_data_buf_alloced && (cmd->tgt_i_sg != NULL)) { + cmd->sg = cmd->tgt_i_sg; + cmd->sg_cnt = cmd->tgt_i_sg_cnt; TRACE_MEM("Tgt used for INQUIRY for not supported " "LUN for cmd %p", cmd); goto go; @@ -4288,7 +4294,7 @@ int scst_acg_remove_name(struct scst_acg *acg, const char *name, bool reassign) static struct scst_cmd *scst_create_prepare_internal_cmd( struct scst_cmd *orig_cmd, const uint8_t *cdb, - unsigned int cdb_len, int bufsize) + unsigned int cdb_len, enum scst_cmd_queue_type queue_type) { struct scst_cmd *res; int rc; @@ -4311,10 +4317,8 @@ static struct scst_cmd *scst_create_prepare_internal_cmd( res->tgt_dev = orig_cmd->tgt_dev; res->cur_order_data = orig_cmd->tgt_dev->curr_order_data; res->lun = orig_cmd->lun; - res->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE; + res->queue_type = queue_type; res->data_direction = SCST_DATA_UNKNOWN; - res->orig_cmd = orig_cmd; - res->bufflen = bufsize; scst_sess_get(res->sess); if (res->tgt_dev != NULL) @@ -4349,13 +4353,14 @@ int scst_prepare_request_sense(struct scst_cmd *orig_cmd) rs_cmd = scst_create_prepare_internal_cmd(orig_cmd, request_sense, sizeof(request_sense), - SCST_SENSE_BUFFERSIZE); + SCST_CMD_QUEUE_HEAD_OF_QUEUE); if (rs_cmd == NULL) goto out_error; + rs_cmd->tgt_i_priv = orig_cmd; + rs_cmd->cdb[1] |= scst_get_cmd_dev_d_sense(orig_cmd); - rs_cmd->data_direction = SCST_DATA_READ; - rs_cmd->expected_data_direction = rs_cmd->data_direction; + rs_cmd->expected_data_direction = SCST_DATA_READ; rs_cmd->expected_transfer_len = SCST_SENSE_BUFFERSIZE; rs_cmd->expected_values_set = 1; @@ -4377,7 +4382,7 @@ out_error: static void scst_complete_request_sense(struct scst_cmd *req_cmd) { - struct scst_cmd *orig_cmd = req_cmd->orig_cmd; + struct scst_cmd *orig_cmd = req_cmd->tgt_i_priv; uint8_t *buf; int len; @@ -4414,6 +4419,330 @@ static void scst_complete_request_sense(struct scst_cmd *req_cmd) return; } +struct scst_i_finish_t { + void (*scst_i_finish_fn) (struct scst_cmd *cmd); +}; + +#define SCST_WRITE_SAME_MAX_EACH_SIZE (128*1024) +#define SCST_WRITE_SAME_MAX_IN_FLIGHT_COMMANDS 32 + +struct scst_write_same_priv { + /* Must be the first for scst_finish_internal_cmd()! */ + struct scst_i_finish_t ws_finish_fn; + + struct scst_cmd *ws_orig_cmd; + + struct mutex ws_mutex; + + int ws_cur_lba; /* in blocks */ + int ws_left_to_send; /* in blocks */ + + int ws_max_each;/* in blocks */ + int ws_cur_in_flight; + + struct scatterlist *ws_sg; + int ws_sg_cnt; +}; + +/* ws_mutex suppose to be locked */ +static int scst_ws_push_single_write(struct scst_write_same_priv *wsp, + int64_t lba, int blocks) +{ + struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; + struct scatterlist *ws_sg = wsp->ws_sg; + int ws_sg_cnt = wsp->ws_sg_cnt; + int res, i; + static uint8_t write16_cdb[16]; + struct scatterlist *sg; + int sg_cnt, len = blocks << ws_cmd->dev->block_shift; + struct scst_cmd *cmd; + int64_t cur_lba; + + TRACE_ENTRY(); + + if (unlikely(test_bit(SCST_CMD_ABORTED, &ws_cmd->cmd_flags)) || + unlikely(ws_cmd->completed)) { + TRACE_DBG("ws cmd %p aborted or completed (%d), aborting " + "further write commands", ws_cmd, ws_cmd->completed); + wsp->ws_left_to_send = 0; + res = -EPIPE; + goto out; + } + + memset(write16_cdb, 0, sizeof(write16_cdb)); + write16_cdb[0] = WRITE_16; + put_unaligned_be64(lba, &write16_cdb[2]); + put_unaligned_be32(blocks, &write16_cdb[10]); + + cmd = scst_create_prepare_internal_cmd(ws_cmd, write16_cdb, + sizeof(write16_cdb), SCST_CMD_QUEUE_SIMPLE); + if (cmd == NULL) { + res = -ENOMEM; + goto out_busy; + } + + cmd->expected_data_direction = SCST_DATA_WRITE; + cmd->expected_transfer_len = len; + cmd->expected_values_set = 1; + + cmd->tgt_i_priv = wsp; + + if ((ws_cmd->cdb[1] & 0x6) == 0) { + TRACE_DBG("Using direct ws_sg %p (cnt %d)", ws_sg, ws_sg_cnt); + sg = ws_sg; + sg_cnt = ws_sg_cnt; + goto set_add; + } + + sg = scst_alloc_sg(len, GFP_KERNEL, &sg_cnt); + if (sg == NULL) { + PRINT_ERROR("Unable to alloc sg for %d blocks", blocks); + res = -ENOMEM; + goto out_destroy; + } + + sg_copy(sg, ws_sg, ws_sg_cnt, len, KM_USER0, KM_USER1); + + cur_lba = lba; + for (i = 0; i < sg_cnt; i++) { + int cur_offs = 0; + while (cur_offs < sg[i].length) { + ((int32_t *)(page_address(sg_page(&sg[i]))))[cur_offs] = cur_lba; + cur_offs += ws_cmd->dev->block_size; + cur_lba++; + } + } + +set_add: + cmd->tgt_i_sg = sg; + cmd->tgt_i_sg_cnt = sg_cnt; + cmd->tgt_i_data_buf_alloced = 1; + + wsp->ws_cur_lba += blocks; + wsp->ws_left_to_send -= blocks; + wsp->ws_cur_in_flight++; + + TRACE_DBG("Adding WRITE(16) cmd %p to head of active cmd list", cmd); + spin_lock_irq(&cmd->cmd_threads->cmd_list_lock); + list_add_tail(&cmd->cmd_list_entry, &cmd->cmd_threads->active_cmd_list); + spin_unlock_irq(&cmd->cmd_threads->cmd_list_lock); + + res = 0; + +out: + TRACE_EXIT_RES(res); + return res; + +out_destroy: + scst_destroy_put_cmd(cmd); + +out_busy: + scst_set_busy(ws_cmd); + goto out; +} + +static void scst_ws_finished(struct scst_write_same_priv *wsp) +{ + struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; + + TRACE_ENTRY(); + + TRACE_DBG("ws cmd %p finished with status %d", ws_cmd, ws_cmd->status); + + sBUG_ON(wsp->ws_cur_in_flight != 0); + + kfree(wsp->ws_sg); + kfree(wsp); + + ws_cmd->completed = 1; /* for success */ + ws_cmd->scst_cmd_done(ws_cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD); + + TRACE_EXIT(); + return; +} + +/* Must be called in a thread context and no locks */ +static void scst_ws_write_cmd_finished(struct scst_cmd *cmd) +{ + struct scst_write_same_priv *wsp = cmd->tgt_i_priv; + struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; + int rc, blocks; + + TRACE_ENTRY(); + + TRACE_DBG("Write cmd %p finished (ws cmd %p, ws_cur_in_flight %d)", + cmd, ws_cmd, wsp->ws_cur_in_flight); + + if ((ws_cmd->cdb[1] & 0x6) != 0) + scst_free_sg(cmd->sg, cmd->sg_cnt); + + cmd->sg = NULL; + cmd->sg_cnt = 0; + + EXTRACHECKS_BUG_ON(!cmd->completed); + + mutex_lock(&wsp->ws_mutex); + + wsp->ws_cur_in_flight--; + + if (cmd->status != 0) { + int rc; + TRACE_DBG("Write cmd %p (ws cmd %p) finished not successfully", + cmd, ws_cmd); + sBUG_ON(cmd->resp_data_len != 0); + if (cmd->status == SAM_STAT_CHECK_CONDITION) + rc = scst_set_cmd_error_sense(ws_cmd, cmd->sense, + cmd->sense_buflen); + else { + sBUG_ON(cmd->sense != NULL); + rc = scst_set_cmd_error_status(ws_cmd, cmd->status); + } + if (rc != 0) { + /* Requeue possible UA */ + if (scst_is_ua_sense(cmd->sense, cmd->sense_buflen)) + scst_requeue_ua(cmd); + } + } + + if (wsp->ws_left_to_send == 0) + goto out_check_finished; + + blocks = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); + + rc = scst_ws_push_single_write(wsp, wsp->ws_cur_lba, blocks); + if (rc != 0) + goto out_check_finished; + + wake_up(&ws_cmd->cmd_threads->cmd_list_waitQ); + +out_unlock: + mutex_unlock(&wsp->ws_mutex); + +out: + TRACE_EXIT(); + return; + +out_check_finished: + if (wsp->ws_cur_in_flight > 0) + goto out_unlock; + + mutex_unlock(&wsp->ws_mutex); + scst_ws_finished(wsp); + goto out; +} + +/* Must be called in a thread context and no locks */ +static void scst_ws_gen_writes(struct scst_write_same_priv *wsp) +{ + struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; + int cnt = 0; + + TRACE_ENTRY(); + + mutex_lock(&wsp->ws_mutex); + + while ((wsp->ws_left_to_send > 0) && + (wsp->ws_cur_in_flight < SCST_WRITE_SAME_MAX_IN_FLIGHT_COMMANDS)) { + int rc, blocks; + + blocks = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); + + rc = scst_ws_push_single_write(wsp, wsp->ws_cur_lba, blocks); + if (rc != 0) + goto out_err; + + cnt++; + } + +out_wake: + if (cnt != 0) + wake_up(&ws_cmd->cmd_threads->cmd_list_waitQ); + + mutex_unlock(&wsp->ws_mutex); + +out: + TRACE_EXIT(); + return; + +out_err: + if (wsp->ws_cur_in_flight != 0) + goto out_wake; + else { + mutex_unlock(&wsp->ws_mutex); + scst_ws_finished(wsp); + goto out; + } +} + +/* + * Library function to perform WRITE SAME in a generic manner. On exit, cmd + * always completed with sense set, if necessary. + */ +void scst_write_same(struct scst_cmd *cmd) +{ + struct scst_write_same_priv *wsp; + int i; + + TRACE_ENTRY(); + + if (cmd->sg_cnt != 1) { + PRINT_ERROR("WRITE SAME must contain only single block of data " + "in a single SG (cmd %p)", cmd); + scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_parameter_value_invalid)); + goto out_done; + } + + if (((cmd->cdb[1] & 0x6) == 0x6) || ((cmd->cdb[1] & 0xE0) != 0)) { + scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb)); + goto out_done; + } + + wsp = kzalloc(sizeof(*wsp), GFP_KERNEL); + if (wsp == NULL) { + PRINT_ERROR("Unable to allocate ws_priv (size %zd, cmd %p)", + sizeof(*wsp), cmd); + goto out_busy; + } + + mutex_init(&wsp->ws_mutex); + wsp->ws_finish_fn.scst_i_finish_fn = scst_ws_write_cmd_finished; + wsp->ws_orig_cmd = cmd; + + wsp->ws_cur_lba = cmd->lba; + wsp->ws_left_to_send = cmd->data_len >> cmd->dev->block_shift; + wsp->ws_max_each = SCST_WRITE_SAME_MAX_EACH_SIZE >> cmd->dev->block_shift; + + wsp->ws_sg_cnt = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); + wsp->ws_sg = kmalloc(wsp->ws_sg_cnt * sizeof(*wsp->ws_sg), GFP_KERNEL); + if (wsp->ws_sg == NULL) { + PRINT_ERROR("Unable to alloc sg for %d entries", wsp->ws_sg_cnt); + goto out_free; + } + sg_init_table(wsp->ws_sg, wsp->ws_sg_cnt); + + for (i = 0; i < wsp->ws_sg_cnt; i++) { + sg_set_page(&wsp->ws_sg[i], sg_page(cmd->sg), + cmd->sg->length, cmd->sg->offset); + } + + scst_ws_gen_writes(wsp); + +out: + TRACE_EXIT(); + return; + +out_free: + kfree(wsp); + +out_busy: + scst_set_busy(cmd); + +out_done: + cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD); + goto out; +} +EXPORT_SYMBOL_GPL(scst_write_same); + int scst_finish_internal_cmd(struct scst_cmd *cmd) { int res; @@ -4422,13 +4751,25 @@ int scst_finish_internal_cmd(struct scst_cmd *cmd) sBUG_ON(!cmd->internal); + if (scst_cmd_atomic(cmd)) { + TRACE_DBG("Rescheduling finished internal atomic cmd %p in a " + "thread context", cmd); + res = SCST_CMD_STATE_RES_NEED_THREAD; + goto out; + } + if (cmd->cdb[0] == REQUEST_SENSE) scst_complete_request_sense(cmd); + else if (cmd->cdb[0] == WRITE_16) { + struct scst_i_finish_t *f = cmd->tgt_i_priv; + f->scst_i_finish_fn(cmd); + } __scst_cmd_put(cmd); res = SCST_CMD_STATE_RES_CONT_NEXT; +out: TRACE_EXIT_HRES(res); return res; } @@ -4829,7 +5170,7 @@ void scst_free_cmd(struct scst_cmd *cmd) * Target driver can already free sg buffer before calling * scst_tgt_cmd_done(). E.g., scst_local has to do that. */ - if (!cmd->tgt_data_buf_alloced) + if (!cmd->tgt_i_data_buf_alloced) scst_check_restore_sg_buff(cmd); if ((cmd->tgtt->on_free_cmd != NULL) && likely(!cmd->internal)) { @@ -5100,7 +5441,7 @@ static void scst_release_space(struct scst_cmd *cmd) if (cmd->sgv == NULL) { if ((cmd->sg != NULL) && - !(cmd->tgt_data_buf_alloced || cmd->dh_data_buf_alloced)) { + !(cmd->tgt_i_data_buf_alloced || cmd->dh_data_buf_alloced)) { TRACE_MEM("Freeing sg %p for cmd %p (cnt %d)", cmd->sg, cmd, cmd->sg_cnt); scst_free_sg(cmd->sg, cmd->sg_cnt); @@ -5109,7 +5450,7 @@ static void scst_release_space(struct scst_cmd *cmd) goto out; } - if (cmd->tgt_data_buf_alloced || cmd->dh_data_buf_alloced) { + if (cmd->tgt_i_data_buf_alloced || cmd->dh_data_buf_alloced) { TRACE_MEM("%s", "*data_buf_alloced set, returning"); goto out; } @@ -5403,7 +5744,7 @@ EXPORT_SYMBOL(scst_scsi_exec_async); /** * scst_copy_sg() - copy data between the command's SGs * - * Copies data between cmd->tgt_sg and cmd->sg in direction defined by + * Copies data between cmd->tgt_i_sg and cmd->sg in direction defined by * copy_dir parameter. */ void scst_copy_sg(struct scst_cmd *cmd, enum scst_sg_copy_dir copy_dir) @@ -5416,7 +5757,7 @@ void scst_copy_sg(struct scst_cmd *cmd, enum scst_sg_copy_dir copy_dir) if (copy_dir == SCST_SG_COPY_FROM_TARGET) { if (cmd->data_direction != SCST_DATA_BIDI) { - src_sg = cmd->tgt_sg; + src_sg = cmd->tgt_i_sg; dst_sg = cmd->sg; to_copy = cmd->bufflen; } else { @@ -5427,7 +5768,7 @@ void scst_copy_sg(struct scst_cmd *cmd, enum scst_sg_copy_dir copy_dir) } } else { src_sg = cmd->sg; - dst_sg = cmd->tgt_sg; + dst_sg = cmd->tgt_i_sg; to_copy = cmd->resp_data_len; } @@ -5925,6 +6266,24 @@ static int get_cdb_info_lba_8_len_4(struct scst_cmd *cmd, return 0; } +static int get_cdb_info_write_same10(struct scst_cmd *cmd, + const struct scst_sdbops *sdbops) +{ + cmd->lba = get_unaligned_be32(cmd->cdb + sdbops->info_lba_off); + cmd->bufflen = 1; + cmd->data_len = get_unaligned_be16(cmd->cdb + sdbops->info_len_off); + return 0; +} + +static int get_cdb_info_write_same16(struct scst_cmd *cmd, + const struct scst_sdbops *sdbops) +{ + cmd->lba = get_unaligned_be64(cmd->cdb + sdbops->info_lba_off); + cmd->bufflen = 1; + cmd->data_len = get_unaligned_be32(cmd->cdb + sdbops->info_len_off); + return 0; +} + /** * scst_get_cdb_info() - fill various info about the command's CDB * @@ -7200,14 +7559,7 @@ bool __scst_check_blocked_dev(struct scst_cmd *cmd) TRACE_ENTRY(); EXTRACHECKS_BUG_ON(cmd->unblock_dev); - - if (unlikely(cmd->internal) && (cmd->cdb[0] == REQUEST_SENSE)) { - /* - * The original command can already block the device, so - * REQUEST SENSE command should always pass. - */ - goto out; - } + EXTRACHECKS_BUG_ON(cmd->internal); if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) goto out; diff --git a/scst/src/scst_targ.c b/scst/src/scst_targ.c index 6b6977155..a4a836955 100644 --- a/scst/src/scst_targ.c +++ b/scst/src/scst_targ.c @@ -107,12 +107,23 @@ static bool scst_check_blocked_dev(struct scst_cmd *cmd) bool res; struct scst_device *dev = cmd->dev; + TRACE_ENTRY(); + + if (unlikely(cmd->internal)) { + /* + * The original command can already block the device and must + * hold reference to it, so internal command should always pass. + */ + sBUG_ON(dev->on_dev_cmd_count == 0); + res = false; + goto out; + } + spin_lock_bh(&dev->dev_lock); dev->on_dev_cmd_count++; cmd->dec_on_dev_needed = 1; - TRACE_DBG("New inc on_dev_count %d (cmd %p)", dev->on_dev_cmd_count, - cmd); + TRACE_DBG("New inc on_dev_count %d (cmd %p)", dev->on_dev_cmd_count, cmd); scst_inc_pr_readers_count(cmd, true); @@ -135,6 +146,8 @@ static bool scst_check_blocked_dev(struct scst_cmd *cmd) spin_unlock_bh(&dev->dev_lock); +out: + TRACE_EXIT_RES(res); return res; } @@ -506,13 +519,14 @@ int scst_pre_parse(struct scst_cmd *cmd) #endif TRACE_DBG("op_name <%s> (cmd %p), direction=%d " - "(expected %d, set %s), bufflen=%d, data_len %d, out_bufflen=%d " - "(expected len %d, out expected len %d), flags=0x%x", cmd->op_name, - cmd, cmd->data_direction, cmd->expected_data_direction, + "(expected %d, set %s), lba %lld, bufflen=%d, data_len %d, " + "out_bufflen=%d (expected len %d, out expected len %d), " + "flags=0x%x", cmd->op_name, cmd, cmd->data_direction, + cmd->expected_data_direction, scst_cmd_is_expected_set(cmd) ? "yes" : "no", - cmd->bufflen, cmd->data_len, cmd->out_bufflen, - cmd->expected_transfer_len, cmd->expected_out_transfer_len, - cmd->op_flags); + (long long)cmd->lba, cmd->bufflen, cmd->data_len, + cmd->out_bufflen, cmd->expected_transfer_len, + cmd->expected_out_transfer_len, cmd->op_flags); res = 0; @@ -811,13 +825,15 @@ set_res: } TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d " - "(expected %d, set %s), bufflen=%d, data len %d, out_bufflen=%d, " - "(expected len %d, out expected len %d), flags=0x%x, lba=%lld", - cmd->op_name, cmd, cmd->data_direction, cmd->expected_data_direction, + "(expected %d, set %s), lba=%lld, bufflen=%d, data len %d, " + "out_bufflen=%d, (expected len %d, out expected len %d), " + "flags=0x%x", cmd->op_name, cmd, cmd->data_direction, + cmd->expected_data_direction, scst_cmd_is_expected_set(cmd) ? "yes" : "no", + (unsigned long long)cmd->lba, cmd->bufflen, cmd->data_len, cmd->out_bufflen, cmd->expected_transfer_len, cmd->expected_out_transfer_len, - cmd->op_flags, (unsigned long long)cmd->lba); + cmd->op_flags); #ifdef CONFIG_SCST_EXTRACHECKS switch (state) { @@ -1033,7 +1049,7 @@ static int scst_prepare_space(struct scst_cmd *cmd) goto alloc; } - cmd->tgt_data_buf_alloced = 1; + cmd->tgt_i_data_buf_alloced = 1; if (unlikely(orig_bufflen < cmd->bufflen)) { PRINT_ERROR("Target driver allocated data " @@ -1042,28 +1058,28 @@ static int scst_prepare_space(struct scst_cmd *cmd) cmd->bufflen); goto out_error; } - TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd); + TRACE_MEM("tgt_i_data_buf_alloced (cmd %p)", cmd); } else goto check; } alloc: - if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) { + if (!cmd->tgt_i_data_buf_alloced && !cmd->dh_data_buf_alloced) { r = scst_alloc_space(cmd); - } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) { + } else if (cmd->dh_data_buf_alloced && !cmd->tgt_i_data_buf_alloced) { TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd); r = 0; - } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) { - TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd); - cmd->sg = cmd->tgt_sg; - cmd->sg_cnt = cmd->tgt_sg_cnt; + } else if (cmd->tgt_i_data_buf_alloced && !cmd->dh_data_buf_alloced) { + TRACE_MEM("tgt_i_data_buf_alloced set (cmd %p)", cmd); + cmd->sg = cmd->tgt_i_sg; + cmd->sg_cnt = cmd->tgt_i_sg_cnt; cmd->out_sg = cmd->tgt_out_sg; cmd->out_sg_cnt = cmd->tgt_out_sg_cnt; r = 0; } else { TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, " - "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg, - cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt); + "sg_cnt %d, tgt_i_sg %p, tgt_i_sg_cnt %d)", cmd, cmd->sg, + cmd->sg_cnt, cmd->tgt_i_sg, cmd->tgt_i_sg_cnt); r = 0; } @@ -1495,9 +1511,9 @@ static int scst_tgt_pre_exec(struct scst_cmd *cmd) } else if (cmd->tgt_out_sg != NULL) { sg = cmd->tgt_out_sg; sg_cnt = cmd->tgt_out_sg_cnt; - } else if (cmd->tgt_sg != NULL) { - sg = cmd->tgt_sg; - sg_cnt = cmd->tgt_sg_cnt; + } else if (cmd->tgt_i_sg != NULL) { + sg = cmd->tgt_i_sg; + sg_cnt = cmd->tgt_i_sg_cnt; } else { sg = cmd->sg; sg_cnt = cmd->sg_cnt; @@ -2331,6 +2347,15 @@ int __scst_check_local_events(struct scst_cmd *cmd, bool preempt_tests_only) TRACE_ENTRY(); + if (unlikely(cmd->internal)) { + /* + * The original command passed all checks and not finished yet + */ + sBUG_ON(cmd->dec_pr_readers_count_needed); + res = 0; + goto out; + } + /* * There's no race here, because we need to trace commands sent * *after* dev_double_ua_possible flag was set. @@ -3504,9 +3529,9 @@ static int scst_xmit_response(struct scst_cmd *cmd) (cmd->data_direction & SCST_DATA_READ)) { int i, j, sg_cnt; struct scatterlist *sg; - if (cmd->tgt_sg != NULL) { - sg = cmd->tgt_sg; - sg_cnt = cmd->tgt_sg_cnt; + if (cmd->tgt_i_sg != NULL) { + sg = cmd->tgt_i_sg; + sg_cnt = cmd->tgt_i_sg_cnt; } else { sg = cmd->sg; sg_cnt = cmd->sg_cnt; @@ -4297,8 +4322,6 @@ void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic) case SCST_CMD_STATE_FINISHED_INTERNAL: res = scst_finish_internal_cmd(cmd); - EXTRACHECKS_BUG_ON(res == - SCST_CMD_STATE_RES_NEED_THREAD); break; default: