From 0b4fe40d3e68dbdd1c2dabe77fcfdce575c520c7 Mon Sep 17 00:00:00 2001 From: Vladislav Bolkhovitin Date: Fri, 1 May 2015 02:04:39 +0000 Subject: [PATCH] scst_lib: Improve WRITE SAME performance Some storage devices have a block size of 512 bytes and a minimum I/O size and/or optimal I/O size of 4096 bytes. Improve WRITE SAME performance for such devices by increasing the I/O granularity from 512 bytes to 4096 bytes. Based on a patch written by Vishal Tripathi and Sushil Sharma . Signed-off-by: Bart Van Assche + with some minor changes git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@6189 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- scst/include/scst.h | 4 ++ scst/src/scst_lib.c | 123 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 116 insertions(+), 11 deletions(-) diff --git a/scst/include/scst.h b/scst/include/scst.h index 20e2c6685..4045f041b 100644 --- a/scst/include/scst.h +++ b/scst/include/scst.h @@ -4080,6 +4080,10 @@ static inline void *sg_virt(struct scatterlist *sg) return page_address(sg_page(sg)) + sg->offset; } +static inline void sg_mark_end(struct scatterlist *sg) +{ +} + #ifndef __BACKPORT_LINUX_SCATTERLIST_H_TO_2_6_23__ static inline void sg_init_table(struct scatterlist *sgl, unsigned int nents) diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index 669cafdb7..ca608d657 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -5165,6 +5165,20 @@ struct scst_write_same_priv { struct scatterlist *ws_sg; }; +#ifdef CONFIG_SCST_EXTRACHECKS +static u64 sg_data_length(struct scatterlist *sgl, int nr) +{ + struct scatterlist *sg; + u64 len = 0; + int i; + + for_each_sg(sgl, sg, nr, i) + len += sg->length; + + return len; +} +#endif + /* ws_mutex suppose to be locked */ static int scst_ws_push_single_write(struct scst_write_same_priv *wsp, int64_t lba, int blocks) @@ -5178,7 +5192,7 @@ static int scst_ws_push_single_write(struct scst_write_same_priv *wsp, TRACE_ENTRY(); - EXTRACHECKS_BUG_ON(blocks > wsp->ws_sg_cnt); + EXTRACHECKS_BUG_ON(len != sg_data_length(wsp->ws_sg, wsp->ws_sg_cnt)); if (unlikely(test_bit(SCST_CMD_ABORTED, &ws_cmd->cmd_flags)) || unlikely(ws_cmd->completed)) { @@ -5208,7 +5222,7 @@ static int scst_ws_push_single_write(struct scst_write_same_priv *wsp, cmd->tgt_i_priv = wsp; cmd->tgt_i_sg = ws_sg; - cmd->tgt_i_sg_cnt = blocks; + cmd->tgt_i_sg_cnt = wsp->ws_sg_cnt; cmd->tgt_i_data_buf_alloced = 1; wsp->ws_cur_lba += blocks; @@ -5241,6 +5255,8 @@ static void scst_ws_finished(struct scst_write_same_priv *wsp) sBUG_ON(wsp->ws_cur_in_flight != 0); + if (sg_page(&wsp->ws_sg[0]) != sg_page(ws_cmd->sg)) + __free_page(sg_page(&wsp->ws_sg[0])); kfree(wsp->ws_sg); kfree(wsp); @@ -5251,6 +5267,51 @@ static void scst_ws_finished(struct scst_write_same_priv *wsp) return; } +/* + * If there is a tail with fewer segments than ws_max_each, adjust the SG + * vector and submit a WRITE command for the tail after all other in-flight + * commands have finished. + */ +static void scst_ws_process_tail(struct scst_write_same_priv *wsp) +{ + struct scst_cmd *ws_cmd = wsp->ws_orig_cmd; + struct scatterlist *sg; + unsigned left; + int i; + + TRACE_ENTRY(); + + lockdep_assert_held(&wsp->ws_mutex); + EXTRACHECKS_BUG_ON(wsp->ws_cur_in_flight > 0); + EXTRACHECKS_BUG_ON(wsp->ws_left_to_send >= wsp->ws_max_each); + + wsp->ws_max_each = wsp->ws_left_to_send; + left = wsp->ws_left_to_send << ws_cmd->dev->block_shift; + for_each_sg(wsp->ws_sg, sg, wsp->ws_sg_cnt, i) { + u32 len = min(left, sg->length); + + if (sg->length > len) { + TRACE_DBG("Processing WS tail of %d << %d = %d bytes - adjusted length of element %d from %d to %d", + wsp->ws_left_to_send, + ws_cmd->dev->block_shift, + wsp->ws_left_to_send << ws_cmd->dev->block_shift, + i, sg->length, len); + sg->length = len; + sg_mark_end(sg); +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) + /* Old versions of sg_mark_end() clear page_link. */ + BUG_ON(sg_page(sg) == NULL); +#endif + wsp->ws_sg_cnt = i + 1; + break; + } + left -= len; + } + + TRACE_EXIT(); + return; +} + /* Must be called in a thread context and no locks */ static void scst_ws_write_cmd_finished(struct scst_cmd *cmd) { @@ -5292,6 +5353,13 @@ static void scst_ws_write_cmd_finished(struct scst_cmd *cmd) if (wsp->ws_left_to_send == 0) goto out_check_finish; + if (wsp->ws_left_to_send < wsp->ws_max_each) { + if (wsp->ws_cur_in_flight > 0) + goto out_check_finish; + else + scst_ws_process_tail(wsp); + } + blocks = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); rc = scst_ws_push_single_write(wsp, wsp->ws_cur_lba, blocks); @@ -5326,13 +5394,15 @@ static void scst_ws_gen_writes(struct scst_write_same_priv *wsp) mutex_lock(&wsp->ws_mutex); - while ((wsp->ws_left_to_send > 0) && - (wsp->ws_cur_in_flight < SCST_MAX_IN_FLIGHT_INTERNAL_COMMANDS)) { - int rc, blocks; + if (wsp->ws_left_to_send < wsp->ws_max_each) + scst_ws_process_tail(wsp); - blocks = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); + while (wsp->ws_left_to_send >= wsp->ws_max_each && + wsp->ws_cur_in_flight < SCST_MAX_IN_FLIGHT_INTERNAL_COMMANDS) { + int rc; - rc = scst_ws_push_single_write(wsp, wsp->ws_cur_lba, blocks); + rc = scst_ws_push_single_write(wsp, wsp->ws_cur_lba, + wsp->ws_max_each); if (rc != 0) goto out_err; @@ -5366,6 +5436,9 @@ out_err: void scst_write_same(struct scst_cmd *cmd) { struct scst_write_same_priv *wsp; + struct page *pg = NULL; + struct scatterlist *sg; + unsigned int offset, length, mult, ws_sg_blocks, left; int i; TRACE_ENTRY(); @@ -5413,18 +5486,44 @@ void scst_write_same(struct scst_cmd *cmd) wsp->ws_left_to_send = cmd->data_len >> cmd->dev->block_shift; wsp->ws_max_each = SCST_MAX_EACH_INTERNAL_IO_SIZE >> cmd->dev->block_shift; - wsp->ws_sg_cnt = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); + if (cmd->bufflen <= PAGE_SIZE / 2) + pg = alloc_page(GFP_KERNEL); + if (pg) { + void *src, *dst; + int k; + + mult = 0; + src = kmap(sg_page(cmd->sg)); + dst = kmap(pg); + for (k = 0; k < PAGE_SIZE; k += cmd->bufflen, mult++) + memcpy(dst + k, src + cmd->sg->offset, cmd->bufflen); + kunmap(pg); + kunmap(src); + offset = 0; + length = k; + } else { + pg = sg_page(cmd->sg); + offset = cmd->sg->offset; + length = cmd->sg->length; + mult = 1; + } + + ws_sg_blocks = min_t(int, wsp->ws_left_to_send, wsp->ws_max_each); + wsp->ws_sg_cnt = (ws_sg_blocks + mult - 1) / mult; wsp->ws_sg = kmalloc(wsp->ws_sg_cnt * sizeof(*wsp->ws_sg), GFP_KERNEL); if (wsp->ws_sg == NULL) { PRINT_ERROR("Unable to alloc sg for %d entries", wsp->ws_sg_cnt); goto out_free; } sg_init_table(wsp->ws_sg, wsp->ws_sg_cnt); + left = ws_sg_blocks << cmd->dev->block_shift; + for_each_sg(wsp->ws_sg, sg, wsp->ws_sg_cnt, i) { + u32 len = min(left, length); - for (i = 0; i < wsp->ws_sg_cnt; i++) { - sg_set_page(&wsp->ws_sg[i], sg_page(cmd->sg), - cmd->sg->length, cmd->sg->offset); + sg_set_page(sg, pg, len, offset); + left -= len; } + sBUG_ON(left != 0); /* crash here to avoid data corruption */ scst_ws_gen_writes(wsp); @@ -5433,6 +5532,8 @@ out: return; out_free: + if (pg && pg != sg_page(cmd->sg)) + __free_page(pg); kfree(wsp); out_busy: