From daee4dcb7ec2aa64215eb90a557de60617545b09 Mon Sep 17 00:00:00 2001 From: Vladislav Bolkhovitin Date: Fri, 6 Nov 2015 03:54:23 +0000 Subject: [PATCH] WRITE SAME command improvements Prepared with help from Sushma Gurram git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@6586 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- scst/include/scst.h | 2 +- scst/include/scst_user.h | 22 +- scst/src/dev_handlers/scst_user.c | 107 +++++++++ scst/src/dev_handlers/scst_vdisk.c | 2 +- scst/src/scst_lib.c | 339 +++++++++++++++++++---------- usr/fileio/common.c | 36 +++ usr/fileio/common.h | 4 + 7 files changed, 396 insertions(+), 116 deletions(-) diff --git a/scst/include/scst.h b/scst/include/scst.h index 32f0376ad..d968aa16f 100644 --- a/scst/include/scst.h +++ b/scst/include/scst.h @@ -5539,7 +5539,7 @@ struct scst_data_descriptor { uint64_t sdd_blocks; }; -void scst_write_same(struct scst_cmd *cmd); +void scst_write_same(struct scst_cmd *cmd, struct scst_data_descriptor *where); __be64 scst_pack_lun(const uint64_t lun, enum scst_lun_addr_method addr_method); uint64_t scst_unpack_lun(const uint8_t *lun, int len); diff --git a/scst/include/scst_user.h b/scst/include/scst_user.h index 0a2e53f92..435efb3d7 100644 --- a/scst/include/scst_user.h +++ b/scst/include/scst_user.h @@ -253,6 +253,15 @@ struct scst_user_scsi_cmd_reply_alloc_mem { aligned_u64 pbuf; }; +/* + * Same as struct scst_data_descriptor, but suitable to pass kernel/user + * space boundary + */ +struct scst_user_data_descriptor { + aligned_u64 usdd_lba; + aligned_u64 usdd_blocks; +}; + /* Be careful adding new members here, this structure is allocated on stack! */ struct scst_user_scsi_cmd_reply_exec { int32_t resp_data_len; @@ -260,11 +269,20 @@ struct scst_user_scsi_cmd_reply_exec { #define SCST_EXEC_REPLY_BACKGROUND 0 #define SCST_EXEC_REPLY_COMPLETED 1 +#define SCST_EXEC_REPLY_DO_WRITE_SAME 2 uint8_t reply_type; uint8_t status; - uint8_t sense_len; - aligned_u64 psense_buffer; + union { + struct { + uint8_t sense_len; + aligned_u64 psense_buffer; + }; + struct { + uint16_t ws_descriptors_len; + aligned_u64 ws_descriptors; + }; + }; }; /* Be careful adding new members here, this structure is allocated on stack! */ diff --git a/scst/src/dev_handlers/scst_user.c b/scst/src/dev_handlers/scst_user.c index dc53e0591..035664925 100644 --- a/scst/src/dev_handlers/scst_user.c +++ b/scst/src/dev_handlers/scst_user.c @@ -1444,6 +1444,110 @@ static int dev_user_process_reply_on_cache_free(struct scst_user_cmd *ucmd) return res; } +static int dev_user_process_ws_reply(struct scst_user_cmd *ucmd, + struct scst_user_scsi_cmd_reply_exec *ereply) +{ + int res = 0, rc, count, i; + struct scst_cmd *cmd = ucmd->cmd; + uint8_t *buf; + struct scst_user_data_descriptor *uwhere; + struct scst_data_descriptor *where; + + TRACE_ENTRY(); + + if (unlikely(cmd->cdb[0] != WRITE_SAME) && + unlikely(cmd->cdb[0] != WRITE_SAME_16)) { + PRINT_ERROR("Request to process WRITE SAME for not WRITE SAME " + "CDB (ucmd %p, cmd %p, op %s)", ucmd, cmd, + scst_get_opcode_name(cmd)); + res = -EINVAL; + goto out_hw_err; + } + + if (unlikely(ereply->status != 0)) { + PRINT_ERROR("Request to process WRITE SAME with not 0 status " + "(ucmd %p, cmd %p, status %d)", ucmd, cmd, ereply->status); + res = -EINVAL; + goto out_hw_err; + } + + if (ereply->ws_descriptors_len == 0) { + scst_write_same(cmd, NULL); + goto out; + } + + if (unlikely(ereply->ws_descriptors_len > PAGE_SIZE)) { + PRINT_ERROR("Too many WRITE SAME descriptors (len %d, ucmd %p, " + "cmd %p)", ereply->ws_descriptors_len, ucmd, cmd); + res = -EOVERFLOW; + goto out_hw_err; + } + + buf = kzalloc(ereply->ws_descriptors_len, GFP_KERNEL); + if (unlikely(buf == NULL)) { + PRINT_ERROR("Unable to alloc WS descriptors buf (size %d)", + ereply->ws_descriptors_len); + goto out_busy; + } + + rc = copy_from_user(buf, + (void __user *)(unsigned long)ereply->ws_descriptors, + ereply->ws_descriptors_len); + if (unlikely(rc != 0)) { + PRINT_ERROR("Failed to copy %d WS descriptors' bytes", rc); + res = -EFAULT; + goto out_free_buf; + } + + uwhere = (struct scst_user_data_descriptor *)buf; + count = ereply->ws_descriptors_len / sizeof(*uwhere); + if (unlikely((ereply->ws_descriptors_len % sizeof(*uwhere)) != 0) || + unlikely(uwhere[count-1].usdd_blocks != 0)) { + PRINT_ERROR("Invalid WS descriptors (ucmd %p, cmd %p)", + ucmd, cmd); + res = -EINVAL; + goto out_free_buf; + } + + where = kmalloc(sizeof(*where) * count, GFP_KERNEL); + if (unlikely(where == NULL)) { + PRINT_ERROR("Unable to alloc WS descriptors where (size %zd)", + sizeof(*where) * count); + goto out_busy_free_buf; + } + + for (i = 0; i < count; i++) { + where[i].sdd_lba = uwhere[i].usdd_lba; + where[i].sdd_blocks = uwhere[i].usdd_blocks; + } + kfree(buf); + + scst_write_same(cmd, where); + +out: + TRACE_EXIT_RES(res); + return res; + +out_free_buf: + kfree(buf); + +out_hw_err: + scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error)); + +out_compl: + cmd->completed = 1; + cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_DIRECT); + /* !! At this point cmd can be already freed !! */ + goto out; + +out_busy_free_buf: + kfree(buf); + +out_busy: + scst_set_busy(cmd); + goto out_compl; +} + static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd, struct scst_user_reply_cmd *reply) { @@ -1480,6 +1584,9 @@ static int dev_user_process_reply_exec(struct scst_user_cmd *ucmd, ucmd->background_exec = 1; TRACE_DBG("Background ucmd %p", ucmd); goto out_compl; + } else if (ereply->reply_type == SCST_EXEC_REPLY_DO_WRITE_SAME) { + res = dev_user_process_ws_reply(ucmd, ereply); + goto out; } else goto out_inval; diff --git a/scst/src/dev_handlers/scst_vdisk.c b/scst/src/dev_handlers/scst_vdisk.c index f62f71a4a..981bff641 100644 --- a/scst/src/dev_handlers/scst_vdisk.c +++ b/scst/src/dev_handlers/scst_vdisk.c @@ -3905,7 +3905,7 @@ static enum compl_status_e vdisk_exec_write_same(struct vdisk_cmd_params *p) if (cmd->cdb[ctrl_offs] & 0x8) vdisk_exec_write_same_unmap(p); else { - scst_write_same(cmd); + scst_write_same(cmd, NULL); res = RUNNING_ASYNC; } diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index ddb64c354..91130a5e8 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -5320,13 +5320,12 @@ int scst_acg_remove_name(struct scst_acg *acg, const char *name, bool reassign) } #endif -static struct scst_cmd *scst_create_prepare_internal_cmd( - struct scst_cmd *orig_cmd, const uint8_t *cdb, - unsigned int cdb_len, enum scst_cmd_queue_type queue_type) +struct scst_cmd *__scst_create_prepare_internal_cmd(const uint8_t *cdb, + unsigned int cdb_len, enum scst_cmd_queue_type queue_type, + struct scst_tgt_dev *tgt_dev, gfp_t gfp_mask, bool fantom) { struct scst_cmd *res; int rc; - gfp_t gfp_mask = scst_cmd_atomic(orig_cmd) ? GFP_ATOMIC : orig_cmd->cmd_gfp_mask; unsigned long flags; TRACE_ENTRY(); @@ -5335,30 +5334,34 @@ static struct scst_cmd *scst_create_prepare_internal_cmd( if (res == NULL) goto out; - res->cmd_threads = orig_cmd->cmd_threads; - res->sess = orig_cmd->sess; - res->atomic = scst_cmd_atomic(orig_cmd); + res->cmd_threads = tgt_dev->active_cmd_threads; + res->sess = tgt_dev->sess; res->internal = 1; - res->tgtt = orig_cmd->tgtt; - res->tgt = orig_cmd->tgt; - res->dev = orig_cmd->dev; - res->devt = orig_cmd->devt; - res->tgt_dev = orig_cmd->tgt_dev; - res->cur_order_data = orig_cmd->tgt_dev->curr_order_data; - res->lun = orig_cmd->lun; + res->tgtt = tgt_dev->sess->tgt->tgtt; + res->tgt = tgt_dev->sess->tgt; + res->dev = tgt_dev->dev; + res->devt = tgt_dev->dev->handler; + res->tgt_dev = tgt_dev; + res->cur_order_data = tgt_dev->curr_order_data; + res->lun = tgt_dev->lun; res->queue_type = queue_type; res->data_direction = SCST_DATA_UNKNOWN; - /* - * We need to keep it here to be able to abort during TM processing. - * They should be aborted to (1) speed up TM processing and (2) to - * guarantee that after a TM command finished the affected device(s) - * is/are in a quiescent state with all affected commands finished and - * others - blocked. - */ - spin_lock_irqsave(&res->sess->sess_list_lock, flags); - list_add_tail(&res->sess_cmd_list_entry, &res->sess->sess_cmd_list); - spin_unlock_irqrestore(&res->sess->sess_list_lock, flags); + if (!fantom) { + /* + * We need to keep it here to be able to abort during TM + * processing. They should be aborted to (1) speed up TM + * processing and (2) to guarantee that after a TM command + * finished the affected device(s) is/are in a quiescent state + * with all affected commands finished and others - blocked. + * + * Fantom commands are exception, because they don't do any + * real work. + */ + spin_lock_irqsave(&res->sess->sess_list_lock, flags); + list_add_tail(&res->sess_cmd_list_entry, &res->sess->sess_cmd_list); + spin_unlock_irqrestore(&res->sess->sess_list_lock, flags); + } scst_sess_get(res->sess); if (res->tgt_dev != NULL) @@ -5379,6 +5382,27 @@ out: return res; } +static struct scst_cmd *scst_create_prepare_internal_cmd( + struct scst_cmd *orig_cmd, const uint8_t *cdb, + unsigned int cdb_len, enum scst_cmd_queue_type queue_type) +{ + struct scst_cmd *res; + gfp_t gfp_mask = scst_cmd_atomic(orig_cmd) ? GFP_ATOMIC : orig_cmd->cmd_gfp_mask; + + TRACE_ENTRY(); + + res = __scst_create_prepare_internal_cmd(cdb, cdb_len, queue_type, + orig_cmd->tgt_dev, gfp_mask, false); + if (res == NULL) + goto out; + + res->atomic = scst_cmd_atomic(orig_cmd); + +out: + TRACE_EXIT_HRES((unsigned long)res); + return res; +} + static void scst_prelim_finish_internal_cmd(struct scst_cmd *cmd) { unsigned long flags; @@ -5492,6 +5516,11 @@ static void scst_complete_request_sense(struct scst_cmd *req_cmd) return; } +struct scst_ws_sg_tail { + struct scatterlist *sg; + int sg_cnt; +}; + struct scst_write_same_priv { /* Must be the first for scst_finish_internal_cmd()! */ scst_i_finish_fn_t ws_finish_fn; @@ -5500,8 +5529,13 @@ struct scst_write_same_priv { struct mutex ws_mutex; - int64_t ws_cur_lba; /* in blocks */ + /* 0 len terminated */ + struct scst_data_descriptor *ws_descriptors; + + int ws_cur_descr; + int ws_left_to_send; /* in blocks */ + int64_t ws_cur_lba; /* in blocks */ __be16 guard_tag; __be16 app_tag; @@ -5510,10 +5544,12 @@ struct scst_write_same_priv { unsigned int inc_ref_tag:1; int ws_max_each;/* in blocks */ - int ws_cur_in_flight; + int ws_cur_in_flight; /* commands */ - int ws_sg_cnt; - struct scatterlist *ws_sg; + struct scst_ws_sg_tail *ws_sg_tails; + + struct scatterlist *ws_sg_full; + int ws_sg_full_cnt; }; #ifdef CONFIG_SCST_EXTRACHECKS @@ -5536,7 +5572,8 @@ static int scst_ws_push_single_write(struct scst_write_same_priv *wsp, { struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; struct scst_device *dev = ws_cmd->dev; - struct scatterlist *ws_sg = wsp->ws_sg; + struct scatterlist *ws_sg; + int ws_sg_cnt; int res; uint8_t write_cdb[32]; int write_cdb_len; @@ -5546,10 +5583,18 @@ static int scst_ws_push_single_write(struct scst_write_same_priv *wsp, TRACE_ENTRY(); + if (blocks == wsp->ws_max_each) { + ws_sg = wsp->ws_sg_full; + ws_sg_cnt = wsp->ws_sg_full_cnt; + } else { + ws_sg = wsp->ws_sg_tails[wsp->ws_cur_descr].sg; + ws_sg_cnt = wsp->ws_sg_tails[wsp->ws_cur_descr].sg_cnt; + } + #ifdef CONFIG_SCST_EXTRACHECKS - if (len != sg_data_length(wsp->ws_sg, wsp->ws_sg_cnt)) + if (len != sg_data_length(ws_sg, ws_sg_cnt)) WARN_ONCE(true, "lba %lld: %d <> %lld\n", lba, len, - sg_data_length(wsp->ws_sg, wsp->ws_sg_cnt)); + sg_data_length(ws_sg, ws_sg_cnt)); #endif if (unlikely(test_bit(SCST_CMD_ABORTED, &ws_cmd->cmd_flags)) || @@ -5654,7 +5699,7 @@ static int scst_ws_push_single_write(struct scst_write_same_priv *wsp, } cmd->tgt_i_sg = ws_sg; - cmd->tgt_i_sg_cnt = wsp->ws_sg_cnt; + cmd->tgt_i_sg_cnt = ws_sg_cnt; cmd->tgt_i_data_buf_alloced = 1; wsp->ws_cur_lba += blocks; @@ -5683,6 +5728,7 @@ out_busy: static void scst_ws_finished(struct scst_write_same_priv *wsp) { struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; + int i; TRACE_ENTRY(); @@ -5690,9 +5736,21 @@ static void scst_ws_finished(struct scst_write_same_priv *wsp) sBUG_ON(wsp->ws_cur_in_flight != 0); - if (sg_page(&wsp->ws_sg[0]) != sg_page(ws_cmd->sg)) - __free_page(sg_page(&wsp->ws_sg[0])); - kfree(wsp->ws_sg); + if (wsp->ws_sg_full && + sg_page(&wsp->ws_sg_full[0]) != sg_page(ws_cmd->sg)) + __free_page(sg_page(&wsp->ws_sg_full[0])); + else if (wsp->ws_sg_tails && wsp->ws_sg_tails[0].sg_cnt != 0 && + sg_page(&wsp->ws_sg_tails[0].sg[0]) != sg_page(ws_cmd->sg)) + __free_page(sg_page(&wsp->ws_sg_tails[0].sg[0])); + if (wsp->ws_sg_full) + kfree(wsp->ws_sg_full); + if (wsp->ws_sg_tails) { + for (i = 0; wsp->ws_descriptors[i].sdd_blocks != 0; i++) + if (wsp->ws_sg_tails[i].sg_cnt != 0) + kfree(wsp->ws_sg_tails[i].sg); + kfree(wsp->ws_sg_tails); + } + kfree(wsp->ws_descriptors); kfree(wsp); ws_cmd->completed = 1; /* for success */ @@ -5702,51 +5760,6 @@ static void scst_ws_finished(struct scst_write_same_priv *wsp) return; } -/* - * If there is a tail with fewer segments than ws_max_each, adjust the SG - * vector and submit a WRITE command for the tail after all other in-flight - * commands have finished. - */ -static void scst_ws_process_tail(struct scst_write_same_priv *wsp) -{ - struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; - struct scatterlist *sg; - unsigned left; - int i; - - TRACE_ENTRY(); - - lockdep_assert_held(&wsp->ws_mutex); - EXTRACHECKS_BUG_ON(wsp->ws_cur_in_flight > 0); - EXTRACHECKS_BUG_ON(wsp->ws_left_to_send >= wsp->ws_max_each); - - wsp->ws_max_each = wsp->ws_left_to_send; - left = wsp->ws_left_to_send << ws_cmd->dev->block_shift; - for_each_sg(wsp->ws_sg, sg, wsp->ws_sg_cnt, i) { - u32 len = min(left, sg->length); - - if (sg->length > len) { - TRACE_DBG("Processing WS tail of %d << %d = %d bytes - adjusted length of element %d from %d to %d", - wsp->ws_left_to_send, - ws_cmd->dev->block_shift, - wsp->ws_left_to_send << ws_cmd->dev->block_shift, - i, sg->length, len); - sg->length = len; - sg_mark_end(sg); -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) - /* Old versions of sg_mark_end() clear page_link. */ - BUG_ON(sg_page(sg) == NULL); -#endif - wsp->ws_sg_cnt = i + 1; - break; - } - left -= len; - } - - TRACE_EXIT(); - return; -} - /* Must be called in a thread context and no locks */ static void scst_ws_write_cmd_finished(struct scst_cmd *cmd) { @@ -5788,14 +5801,17 @@ static void scst_ws_write_cmd_finished(struct scst_cmd *cmd) } } - if (wsp->ws_left_to_send == 0) - goto out_check_finish; - - if (wsp->ws_left_to_send < wsp->ws_max_each) { - if (wsp->ws_cur_in_flight > 0) + if (wsp->ws_left_to_send == 0) { + if (wsp->ws_descriptors[wsp->ws_cur_descr+1].sdd_blocks != 0) { + wsp->ws_cur_descr++; + wsp->ws_cur_lba = wsp->ws_descriptors[wsp->ws_cur_descr].sdd_lba; + wsp->ws_left_to_send = wsp->ws_descriptors[wsp->ws_cur_descr].sdd_blocks; + TRACE_DBG("wsp %p, cur descr %d, cur lba %lld, left %d", + wsp, wsp->ws_cur_descr, (long long )wsp->ws_cur_lba, + wsp->ws_left_to_send); + } + if (wsp->ws_left_to_send == 0) goto out_check_finish; - else - scst_ws_process_tail(wsp); } blocks = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); @@ -5827,18 +5843,15 @@ static void scst_ws_gen_writes(struct scst_write_same_priv *wsp) { struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; int cnt = 0; + int rc; TRACE_ENTRY(); mutex_lock(&wsp->ws_mutex); - if (wsp->ws_left_to_send < wsp->ws_max_each) - scst_ws_process_tail(wsp); - +again: while (wsp->ws_left_to_send >= wsp->ws_max_each && wsp->ws_cur_in_flight < SCST_MAX_IN_FLIGHT_INTERNAL_COMMANDS) { - int rc; - rc = scst_ws_push_single_write(wsp, wsp->ws_cur_lba, wsp->ws_max_each); if (rc != 0) @@ -5846,6 +5859,26 @@ static void scst_ws_gen_writes(struct scst_write_same_priv *wsp) cnt++; } + if (wsp->ws_left_to_send > 0 && + wsp->ws_cur_in_flight < SCST_MAX_IN_FLIGHT_INTERNAL_COMMANDS) { + rc = scst_ws_push_single_write(wsp, wsp->ws_cur_lba, + wsp->ws_left_to_send); + if (rc != 0) + goto out_err; + + cnt++; + } + + if ((wsp->ws_left_to_send == 0) && + (wsp->ws_descriptors[wsp->ws_cur_descr+1].sdd_blocks != 0)) { + wsp->ws_cur_descr++; + wsp->ws_cur_lba = wsp->ws_descriptors[wsp->ws_cur_descr].sdd_lba; + wsp->ws_left_to_send = wsp->ws_descriptors[wsp->ws_cur_descr].sdd_blocks; + TRACE_DBG("wsp %p, cur descr %d, cur lba %lld, left %d", + wsp, wsp->ws_cur_descr, (long long )wsp->ws_cur_lba, + wsp->ws_left_to_send); + goto again; + } out_wake: if (cnt != 0) @@ -5867,21 +5900,77 @@ out_err: } } +static int scst_ws_sg_init(struct scatterlist **ws_sg, int ws_sg_cnt, + struct page *pg, unsigned int offset, unsigned int length, + unsigned int total_bytes) +{ + struct scatterlist *sg; + int i; + + *ws_sg = kmalloc(ws_sg_cnt * sizeof(**ws_sg), GFP_KERNEL); + if (*ws_sg == NULL) { + PRINT_ERROR("Unable to alloc sg for %d entries", ws_sg_cnt); + return -ENOMEM; + } + sg_init_table(*ws_sg, ws_sg_cnt); + for_each_sg(*ws_sg, sg, ws_sg_cnt, i) { + u32 len = min(total_bytes, length); + sg_set_page(sg, pg, len, offset); + total_bytes -= len; + } + sBUG_ON(total_bytes != 0); /* crash here to avoid data corruption */ + + return 0; +} + +static int scst_ws_sg_tails_get(struct scst_data_descriptor *where, struct scst_write_same_priv *wsp) +{ + int i; + + TRACE_ENTRY(); + + for (i = 0; where[i].sdd_blocks != 0; i++) { + if (where[i].sdd_blocks / wsp->ws_max_each != 0) { + wsp->ws_sg_full_cnt = wsp->ws_max_each; + break; + } + } + while (where[i].sdd_blocks != 0) + i++; + + wsp->ws_sg_tails = kzalloc(sizeof(*(wsp->ws_sg_tails))*(i+1), GFP_KERNEL); + if (wsp->ws_sg_tails == NULL) { + PRINT_ERROR("Unable to allocate ws_sg_tails (size %zd)", + sizeof(*wsp->ws_sg_tails)*i); + return -ENOMEM; + } + for (i = 0; where[i].sdd_blocks != 0; i++) + wsp->ws_sg_tails[i].sg_cnt = where[i].sdd_blocks % wsp->ws_max_each; + + TRACE_EXIT(); + return 0; +} + /* * Library function to perform WRITE SAME in a generic manner. On exit, cmd * always completed with sense set, if necessary. + * + * Parameter "where" is an array of descriptors where to write the same + * block. Last element in this array has len 0. It must be allocated by + * k?alloc() and will be kfree() by this function on finish. */ -void scst_write_same(struct scst_cmd *cmd) +void scst_write_same(struct scst_cmd *cmd, struct scst_data_descriptor *where) { struct scst_write_same_priv *wsp; int i, rc; struct page *pg = NULL; - struct scatterlist *sg; - unsigned int offset, length, mult, ws_sg_blocks, left; + unsigned int offset, length, mult, ws_sg_full_blocks, ws_sg_tail_blocks; uint8_t ctrl_offs = (cmd->cdb_len < 32) ? 1 : 10; TRACE_ENTRY(); + scst_set_exec_time(cmd); + if (unlikely(cmd->data_len <= 0)) { scst_set_invalid_field_in_cdb(cmd, cmd->len_off, 0); goto out_done; @@ -5910,6 +5999,20 @@ void scst_write_same(struct scst_cmd *cmd) goto out_done; } + if (where == NULL) { + where = kzalloc(sizeof(*where) << 1, GFP_KERNEL); + if (where == NULL) { + PRINT_ERROR("Unable to allocate ws_priv (size %zd, cmd %p)", + sizeof(*where) << 1, cmd); + goto out_busy; + } + where->sdd_lba = cmd->lba; + where->sdd_blocks = cmd->data_len >> cmd->dev->block_shift; + } + + if (unlikely(where->sdd_blocks == 0)) + goto out_done; + rc = scst_dif_process_write(cmd); if (unlikely(rc != 0)) goto out_done; @@ -5925,10 +6028,15 @@ void scst_write_same(struct scst_cmd *cmd) wsp->ws_finish_fn = scst_ws_write_cmd_finished; wsp->ws_orig_cmd = cmd; - wsp->ws_cur_lba = cmd->lba; - wsp->ws_left_to_send = cmd->data_len >> cmd->dev->block_shift; + wsp->ws_descriptors = where; + wsp->ws_cur_descr = 0; + wsp->ws_cur_lba = where[0].sdd_lba; + wsp->ws_left_to_send = where[0].sdd_blocks; wsp->ws_max_each = SCST_MAX_EACH_INTERNAL_IO_SIZE >> cmd->dev->block_shift; + if (scst_ws_sg_tails_get(where, wsp) == -ENOMEM) + goto out_busy; + if (cmd->bufflen <= PAGE_SIZE / 2) pg = alloc_page(GFP_KERNEL); if (pg) { @@ -5953,22 +6061,28 @@ void scst_write_same(struct scst_cmd *cmd) mult = 1; } - ws_sg_blocks = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); - wsp->ws_sg_cnt = (ws_sg_blocks + mult - 1) / mult; - wsp->ws_sg = kmalloc(wsp->ws_sg_cnt * sizeof(*wsp->ws_sg), GFP_KERNEL); - if (wsp->ws_sg == NULL) { - PRINT_ERROR("Unable to alloc sg for %d entries", wsp->ws_sg_cnt); - goto out_free; + if (wsp->ws_sg_full_cnt != 0) { + ws_sg_full_blocks = wsp->ws_sg_full_cnt; + wsp->ws_sg_full_cnt = (ws_sg_full_blocks + mult - 1) / mult; + TRACE_DBG("Allocating %d SGs", wsp->ws_sg_full_cnt); + rc = scst_ws_sg_init(&wsp->ws_sg_full, wsp->ws_sg_full_cnt, pg, + offset, length, + ws_sg_full_blocks << cmd->dev->block_shift); + if (rc != 0) + goto out_free; } - sg_init_table(wsp->ws_sg, wsp->ws_sg_cnt); - left = ws_sg_blocks << cmd->dev->block_shift; - for_each_sg(wsp->ws_sg, sg, wsp->ws_sg_cnt, i) { - u32 len = min(left, length); - - sg_set_page(sg, pg, len, offset); - left -= len; + for (i = 0; wsp->ws_descriptors[i].sdd_blocks != 0; i++) { + if (wsp->ws_sg_tails[i].sg_cnt != 0) { + ws_sg_tail_blocks = wsp->ws_sg_tails[i].sg_cnt; + wsp->ws_sg_tails[i].sg_cnt = (ws_sg_tail_blocks + mult - 1) / mult; + TRACE_DBG("Allocating %d tail SGs for descriptor[%d] ", wsp->ws_sg_tails[i].sg_cnt, i); + rc = scst_ws_sg_init(&wsp->ws_sg_tails[i].sg, wsp->ws_sg_tails[i].sg_cnt, pg, + offset, length, + ws_sg_tail_blocks << cmd->dev->block_shift); + if (rc != 0) + goto out_free; + } } - sBUG_ON(left != 0); /* crash here to avoid data corruption */ if (scst_cmd_needs_dif_buf(cmd)) { struct t10_pi_tuple *t; @@ -5991,7 +6105,7 @@ void scst_write_same(struct scst_cmd *cmd) wsp->inc_ref_tag = 0; break; default: - sBUG_ON(1); + sBUG(); break; } @@ -6013,6 +6127,7 @@ out_busy: scst_set_busy(cmd); out_done: + kfree(where); cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_THREAD); goto out; } diff --git a/usr/fileio/common.c b/usr/fileio/common.c index 47b673fcc..41400c44e 100644 --- a/usr/fileio/common.c +++ b/usr/fileio/common.c @@ -47,6 +47,7 @@ static int exec_fsync(struct vdisk_cmd *vcmd); static void exec_read(struct vdisk_cmd *vcmd, loff_t loff); static void exec_write(struct vdisk_cmd *vcmd, loff_t loff); static void exec_verify(struct vdisk_cmd *vcmd, loff_t loff); +static void exec_write_same(struct vdisk_cmd *vcmd); static int open_dev_fd(struct vdisk_dev *dev) { @@ -408,6 +409,10 @@ static int do_exec(struct vdisk_cmd *vcmd) case READ_CAPACITY: exec_read_capacity(vcmd); break; + case WRITE_SAME_10: + case WRITE_SAME_16: + exec_write_same(vcmd); + break; case SERVICE_ACTION_IN_16: if ((cmd->cdb[1] & 0x1f) == SAI_READ_CAPACITY_16) exec_read_capacity16(vcmd); @@ -1761,3 +1766,34 @@ out: TRACE_EXIT(); return; } + +static void exec_write_same(struct vdisk_cmd *vcmd) +{ + struct vdisk_dev *dev = vcmd->dev; + struct scst_user_scsi_cmd_exec *cmd = &vcmd->cmd->exec_cmd; + struct scst_user_scsi_cmd_reply_exec *reply = &vcmd->reply->exec_reply; + uint64_t blocks = cmd->data_len >> dev->block_shift; + static struct scst_user_data_descriptor uwhere[3]; + + TRACE_ENTRY(); + + if (blocks < 2) + goto out; + + /* It's only a debug code, so it's OK to use static descr */ + + memset(uwhere, 0, sizeof(uwhere)); + + uwhere[0].usdd_lba = cmd->lba; + uwhere[0].usdd_blocks = 1; + uwhere[1].usdd_lba = cmd->lba+1; + uwhere[1].usdd_blocks = blocks-1; + + reply->reply_type = SCST_EXEC_REPLY_DO_WRITE_SAME; + reply->ws_descriptors_len = sizeof(uwhere); + reply->ws_descriptors = (unsigned long)uwhere; + +out: + TRACE_EXIT(); + return; +} diff --git a/usr/fileio/common.h b/usr/fileio/common.h index f526bd8ca..8fb1f14aa 100644 --- a/usr/fileio/common.h +++ b/usr/fileio/common.h @@ -23,6 +23,10 @@ #include "debug.h" +#ifndef WRITE_SAME_10 +#define WRITE_SAME_10 0x41 +#endif + /* 8 byte ASCII Vendor */ #define VENDOR "SCST_USR" /* 4 byte ASCII Product Revision Level - left aligned */