diff --git a/scst/include/scst.h b/scst/include/scst.h index d630a77eb..db5c47f9f 100644 --- a/scst/include/scst.h +++ b/scst/include/scst.h @@ -364,7 +364,7 @@ enum scst_exec_context { #define SCST_CMD_CAN_BE_DESTROYED 3 /************************************************************* - ** Tgt_dev's flags (tgt_dev_flags) + ** Tgt_dev's async. flags (tgt_dev_flags) *************************************************************/ /* Set if tgt_dev has Unit Attention sense */ @@ -381,6 +381,8 @@ enum scst_exec_context { #define SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC 9 #define SCST_TGT_DEV_AFTER_EXEC_ATOMIC 10 +#define SCST_TGT_DEV_CLUST_POOL 11 + #ifdef CONFIG_SCST_DEBUG_TM #define SCST_TGT_DEV_UNDER_TM_DBG 20 #endif @@ -1501,7 +1503,7 @@ struct scst_tgt_dev { /* * Set if the prev cmd was ORDERED. Size must allow unprotected - * modifications + * modifications independant to the neighbour fields. */ unsigned long prev_cmd_ordered; diff --git a/scst/include/scst_sgv.h b/scst/include/scst_sgv.h index 7b94fceda..f6488f28a 100644 --- a/scst/include/scst_sgv.h +++ b/scst/include/scst_sgv.h @@ -43,7 +43,15 @@ struct scst_mem_lim { int max_allowed_pages; }; -struct sgv_pool *sgv_pool_create(const char *name, int clustered); +/* Types of clustering */ +enum sgv_clustering_types { + sgv_no_clustering = 0, + sgv_tail_clustering, + sgv_full_clustering, +}; + +struct sgv_pool *sgv_pool_create(const char *name, + enum sgv_clustering_types clustered); void sgv_pool_destroy(struct sgv_pool *pool); void sgv_pool_set_allocator(struct sgv_pool *pool, diff --git a/scst/src/dev_handlers/scst_user.c b/scst/src/dev_handlers/scst_user.c index b9e0c4126..d4cf13eb0 100644 --- a/scst/src/dev_handlers/scst_user.c +++ b/scst/src/dev_handlers/scst_user.c @@ -82,6 +82,7 @@ struct scst_user_dev { struct scst_mem_lim udev_mem_lim; struct sgv_pool *pool; + struct sgv_pool *pool_clust; uint8_t parse_type; uint8_t on_free_cmd_type; @@ -278,12 +279,6 @@ static inline int calc_num_pg(unsigned long buf, int len) return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0); } -static inline int is_need_offs_page(unsigned long buf, int len) -{ - return ((buf & ~PAGE_MASK) != 0) && - ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK)); -} - static void __dev_user_not_reg(void) { PRINT_ERROR("%s", "Device not registered"); @@ -497,6 +492,12 @@ static inline int is_buff_cached(struct scst_user_cmd *ucmd) return 0; } +static inline int is_need_offs_page(unsigned long buf, int len) +{ + return ((buf & ~PAGE_MASK) != 0) && + ((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK)); +} + /* * Returns 0 for success, <0 for fatal failure, >0 - need pages. * Unmaps the buffer, if needed in case of error @@ -510,6 +511,7 @@ static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff) int flags = 0; int bufflen = cmd->bufflen; int last_len = 0; + struct sgv_pool *pool; TRACE_ENTRY(); @@ -539,13 +541,21 @@ static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff) } ucmd->buff_cached = cached_buff; - cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags, - &cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd); + if (test_bit(SCST_TGT_DEV_CLUST_POOL, &cmd->tgt_dev->tgt_dev_flags)) + pool = dev->pool_clust; + else + pool = dev->pool; + + cmd->sg = sgv_pool_alloc((struct sgv_pool *)cmd->tgt_dev->dh_priv, + bufflen, gfp_mask, flags, &cmd->sg_cnt, &ucmd->sgv, + &dev->udev_mem_lim, ucmd); if (cmd->sg != NULL) { struct scst_user_cmd *buf_ucmd = (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv); - TRACE_MEM("Buf ucmd %p", buf_ucmd); + TRACE_MEM("Buf ucmd %p (cmd->sg_cnt %d, last seg len %d, " + "last_len %d, bufflen %d)", buf_ucmd, cmd->sg_cnt, + cmd->sg[cmd->sg_cnt-1].length, last_len, bufflen); ucmd->ubuff = buf_ucmd->ubuff; ucmd->buf_ucmd = buf_ucmd; @@ -554,13 +564,13 @@ static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff) (ucmd != buf_ucmd)); if (last_len != 0) { - /* We don't use clustering, so the assignment is safe */ - cmd->sg[cmd->sg_cnt-1].length = last_len; + cmd->sg[cmd->sg_cnt-1].length &= PAGE_MASK; + cmd->sg[cmd->sg_cnt-1].length += last_len; } TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, " - "last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff, - last_len, cmd->sg[cmd->sg_cnt-1].length); + "last seg len %d)", ucmd, cached_buff, ucmd->ubuff, + cmd->sg[cmd->sg_cnt-1].length); if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) { static int ll; @@ -2311,6 +2321,16 @@ static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev) TRACE_ENTRY(); + /* + * We can't replace tgt_dev->pool, because it can be used to allocate + * memory for SCST local commands, like REPORT LUNS, where there is no + * corresponding ucmd. Otherwise we will crash in dev_user_alloc_sg(). + */ + if (test_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags)) + tgt_dev->dh_priv = dev->pool_clust; + else + tgt_dev->dh_priv = dev->pool; + ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL); if (ucmd == NULL) goto out_nomem; @@ -2554,18 +2574,21 @@ static int dev_user_register_dev(struct file *file, scst_init_mem_lim(&dev->udev_mem_lim); - /* - * We don't use clustered pool, since it implies pages reordering, - * which isn't possible with user space supplied buffers. Although - * it's still possible to cluster pages by the tail of each other, - * seems it doesn't worth the effort. - */ - dev->pool = sgv_pool_create(dev->name, 0); + dev->pool = sgv_pool_create(dev->name, sgv_no_clustering); if (dev->pool == NULL) - goto out_put; + goto out_free_dev; sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages, dev_user_free_sg_entries); + scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s-clust", + dev->name); + dev->pool_clust = sgv_pool_create(dev->devtype.name, + sgv_tail_clustering); + if (dev->pool_clust == NULL) + goto out_free0; + sgv_pool_set_allocator(dev->pool_clust, dev_user_alloc_pages, + dev_user_free_sg_entries); + scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s", dev->name); dev->devtype.type = dev_desc->type; @@ -2645,9 +2668,13 @@ out_del_free: spin_unlock(&dev_list_lock); out_free: + sgv_pool_destroy(dev->pool_clust); + +out_free0: sgv_pool_destroy(dev->pool); + +out_free_dev: kfree(dev); - goto out_put; out_put: module_put(THIS_MODULE); @@ -2840,6 +2867,7 @@ static int dev_user_release(struct inode *inode, struct file *file) scst_unregister_virtual_device(dev->virt_id); scst_unregister_virtual_dev_driver(&dev->devtype); + sgv_pool_destroy(dev->pool_clust); sgv_pool_destroy(dev->pool); TRACE_DBG("Unregistering finished (dev %p)", dev); diff --git a/scst/src/scst_mem.c b/scst/src/scst_mem.c index 304e1b5cf..609e8e3a5 100644 --- a/scst/src/scst_mem.c +++ b/scst/src/scst_mem.c @@ -37,10 +37,16 @@ static struct scst_sgv_pools_manager sgv_pools_mgr; +static inline bool sgv_pool_clustered(const struct sgv_pool *pool) +{ + return (pool->clustering_type != sgv_no_clustering); +} + void scst_sgv_pool_use_norm(struct scst_tgt_dev *tgt_dev) { tgt_dev->gfp_mask = __GFP_NOWARN; tgt_dev->pool = &sgv_pools_mgr.default_set.norm; + clear_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags); } void scst_sgv_pool_use_norm_clust(struct scst_tgt_dev *tgt_dev) @@ -48,6 +54,7 @@ void scst_sgv_pool_use_norm_clust(struct scst_tgt_dev *tgt_dev) TRACE_MEM("%s", "Use clustering"); tgt_dev->gfp_mask = __GFP_NOWARN; tgt_dev->pool = &sgv_pools_mgr.default_set.norm_clust; + set_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags); } void scst_sgv_pool_use_dma(struct scst_tgt_dev *tgt_dev) @@ -55,9 +62,10 @@ void scst_sgv_pool_use_dma(struct scst_tgt_dev *tgt_dev) TRACE_MEM("%s", "Use ISA DMA memory"); tgt_dev->gfp_mask = __GFP_NOWARN | GFP_DMA; tgt_dev->pool = &sgv_pools_mgr.default_set.dma; + clear_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags); } -static int scst_check_clustering(struct scatterlist *sg, int cur, int hint) +static int sgv_check_full_clustering(struct scatterlist *sg, int cur, int hint) { int res = -1; int i = hint; @@ -65,7 +73,8 @@ static int scst_check_clustering(struct scatterlist *sg, int cur, int hint) int len_cur = sg[cur].length; unsigned long pfn_cur_next = pfn_cur + (len_cur >> PAGE_SHIFT); int full_page_cur = (len_cur & (PAGE_SIZE - 1)) == 0; - unsigned long pfn, pfn_next, full_page; + unsigned long pfn, pfn_next; + bool full_page; #if 0 TRACE_MEM("pfn_cur %ld, pfn_cur_next %ld, len_cur %d, full_page_cur %d", @@ -117,6 +126,47 @@ out_head: goto out; } +static int sgv_check_tail_clustering(struct scatterlist *sg, int cur, int hint) +{ + int res = -1; + unsigned long pfn_cur = page_to_pfn(sg_page(&sg[cur])); + int len_cur = sg[cur].length; + int prev; + unsigned long pfn_prev; + bool full_page; + +#ifdef SCST_HIGHMEM + if (page >= highmem_start_page) { + TRACE_MEM("%s", "HIGHMEM page allocated, no clustering") + goto out; + } +#endif + +#if 0 + TRACE_MEM("pfn_cur %ld, pfn_cur_next %ld, len_cur %d, full_page_cur %d", + pfn_cur, pfn_cur_next, len_cur, full_page_cur); +#endif + + if (cur == 0) + goto out; + + prev = cur - 1; + pfn_prev = page_to_pfn(sg_page(&sg[prev])) + + (sg[prev].length >> PAGE_SHIFT); + full_page = (sg[prev].length & (PAGE_SIZE - 1)) == 0; + + if ((pfn_prev == pfn_cur) && full_page) { + TRACE_MEM("SG segment %d will be tail merged with segment %d", + cur, prev); + sg[prev].length += len_cur; + sg_clear(&sg[cur]); + res = prev; + } + +out: + return res; +} + static void scst_free_sys_sg_entries(struct scatterlist *sg, int sg_count, void *priv) { @@ -173,14 +223,15 @@ static struct page *scst_alloc_sys_pages(struct scatterlist *sg, } static int scst_alloc_sg_entries(struct scatterlist *sg, int pages, - gfp_t gfp_mask, int clustered, struct trans_tbl_ent *trans_tbl, + gfp_t gfp_mask, enum sgv_clustering_types clustering_type, + struct trans_tbl_ent *trans_tbl, const struct sgv_pool_alloc_fns *alloc_fns, void *priv) { int sg_count = 0; int pg, i, j; int merged = -1; - TRACE_MEM("pages=%d, clustered=%d", pages, clustered); + TRACE_MEM("pages=%d, clustering_type=%d", pages, clustering_type); #if 0 gfp_mask |= __GFP_COLD; @@ -201,17 +252,22 @@ static int scst_alloc_sg_entries(struct scatterlist *sg, int pages, priv); if (rc == NULL) goto out_no_mem; - if (clustered) { - merged = scst_check_clustering(sg, sg_count, merged); - if (merged == -1) - sg_count++; - } else + + if (clustering_type == sgv_full_clustering) + merged = sgv_check_full_clustering(sg, sg_count, merged); + else if (clustering_type == sgv_tail_clustering) + merged = sgv_check_tail_clustering(sg, sg_count, merged); + else + merged = -1; + + if (merged == -1) sg_count++; + TRACE_MEM("pg=%d, merged=%d, sg_count=%d", pg, merged, sg_count); } - if (clustered && (trans_tbl != NULL)) { + if ((clustering_type != sgv_no_clustering) && (trans_tbl != NULL)) { pg = 0; for (i = 0; i < pages; i++) { int n = (sg[i].length >> PAGE_SHIFT) + @@ -254,7 +310,7 @@ static int sgv_alloc_arrays(struct sgv_pool_obj *obj, sg_init_table(obj->sg_entries, pages_to_alloc); - if (obj->owner_pool->clustered) { + if (sgv_pool_clustered(obj->owner_pool)) { if (order <= sgv_pools_mgr.sgv_max_trans_order) { obj->trans_tbl = (struct trans_tbl_ent *)obj->sg_entries_data; @@ -369,7 +425,7 @@ static void sgv_pool_cached_put(struct sgv_pool_obj *sgv) TRACE_MEM("sgv %p, order %d, sg_count %d", sgv, sgv->order_or_pages, sgv->sg_count); - if (owner->clustered) { + if (sgv_pool_clustered(owner)) { /* Make objects with less entries more preferred */ __list_for_each(entry, list) { struct sgv_pool_obj *tmp = list_entry(entry, @@ -620,7 +676,7 @@ struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size, obj->sg_entries = obj->sg_entries_data; sg_init_table(obj->sg_entries, pages_to_alloc); TRACE_MEM("sg_entries %p", obj->sg_entries); - if (pool->clustered) { + if (sgv_pool_clustered(pool)) { obj->trans_tbl = (struct trans_tbl_ent *) (obj->sg_entries + pages_to_alloc); TRACE_MEM("trans_tbl %p", obj->trans_tbl); @@ -683,8 +739,8 @@ struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size, } obj->sg_count = scst_alloc_sg_entries(obj->sg_entries, - pages_to_alloc, gfp_mask, pool->clustered, obj->trans_tbl, - &pool->alloc_fns, priv); + pages_to_alloc, gfp_mask, pool->clustering_type, + obj->trans_tbl, &pool->alloc_fns, priv); if (unlikely(obj->sg_count <= 0)) { obj->sg_count = 0; if ((flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL) && cache) @@ -714,14 +770,14 @@ success: if (cache) { int sg; atomic_inc(&pool->cache_acc[order].total_alloc); - if (pool->clustered) + if (sgv_pool_clustered(pool)) cnt = obj->trans_tbl[pages-1].sg_num; else cnt = pages; sg = cnt-1; obj->orig_sg = sg; obj->orig_length = obj->sg_entries[sg].length; - if (pool->clustered) { + if (sgv_pool_clustered(pool)) { obj->sg_entries[sg].length = (pages - obj->trans_tbl[sg].pg_count) << PAGE_SHIFT; } @@ -860,8 +916,8 @@ struct scatterlist *scst_alloc(int size, gfp_t gfp_mask, int *count) * scst_free() to figure out how many pages are in the SG vector. * So, always don't use clustering. */ - *count = scst_alloc_sg_entries(res, pages, gfp_mask, 0, NULL, - &sys_alloc_fns, NULL); + *count = scst_alloc_sg_entries(res, pages, gfp_mask, sgv_no_clustering, + NULL, &sys_alloc_fns, NULL); if (*count <= 0) goto out_free; @@ -901,7 +957,8 @@ static void sgv_pool_cached_init(struct sgv_pool *pool) INIT_LIST_HEAD(&pool->recycling_lists[i]); } -int sgv_pool_init(struct sgv_pool *pool, const char *name, int clustered) +int sgv_pool_init(struct sgv_pool *pool, const char *name, + enum sgv_clustering_types clustering_type) { int res = -ENOMEM; int i; @@ -918,12 +975,12 @@ int sgv_pool_init(struct sgv_pool *pool, const char *name, int clustered) atomic_set(&pool->acc.other_merged, 0); atomic_set(&pool->acc.big_merged, 0); - pool->clustered = clustered; + pool->clustering_type = clustering_type; pool->alloc_fns.alloc_pages_fn = scst_alloc_sys_pages; pool->alloc_fns.free_pages_fn = scst_free_sys_sg_entries; - TRACE_MEM("name %s, sizeof(*obj)=%zd, clustered=%d", name, sizeof(*obj), - clustered); + TRACE_MEM("name %s, sizeof(*obj)=%zd, clustering_type=%d", name, + sizeof(*obj), clustering_type); strncpy(pool->name, name, sizeof(pool->name)-1); pool->name[sizeof(pool->name)-1] = '\0'; @@ -938,14 +995,16 @@ int sgv_pool_init(struct sgv_pool *pool, const char *name, int clustered) if (i <= sgv_pools_mgr.sgv_max_local_order) { size = sizeof(*obj) + (1 << i) * (sizeof(obj->sg_entries[0]) + - (clustered ? sizeof(obj->trans_tbl[0]) : 0)); + ((clustering_type != sgv_no_clustering) ? + sizeof(obj->trans_tbl[0]) : 0)); } else if (i <= sgv_pools_mgr.sgv_max_trans_order) { /* * sgv ie sg_entries is allocated outside object, but * ttbl is still embedded. */ size = sizeof(*obj) + (1 << i) * - ((clustered ? sizeof(obj->trans_tbl[0]) : 0)); + (((clustering_type != sgv_no_clustering) ? + sizeof(obj->trans_tbl[0]) : 0)); } else { size = sizeof(*obj); @@ -1070,7 +1129,8 @@ void sgv_pool_set_allocator(struct sgv_pool *pool, } EXPORT_SYMBOL(sgv_pool_set_allocator); -struct sgv_pool *sgv_pool_create(const char *name, int clustered) +struct sgv_pool *sgv_pool_create(const char *name, + enum sgv_clustering_types clustering_type) { struct sgv_pool *pool; int rc; @@ -1083,7 +1143,7 @@ struct sgv_pool *sgv_pool_create(const char *name, int clustered) goto out; } - rc = sgv_pool_init(pool, name, clustered); + rc = sgv_pool_init(pool, name, clustering_type); if (rc != 0) goto out_free; @@ -1207,15 +1267,17 @@ int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark) INIT_LIST_HEAD(&pools->mgr.sorted_recycling_list); spin_lock_init(&pools->mgr.pool_mgr_lock); - res = sgv_pool_init(&pools->default_set.norm, "sgv", 0); + res = sgv_pool_init(&pools->default_set.norm, "sgv", sgv_no_clustering); if (res != 0) goto out; - res = sgv_pool_init(&pools->default_set.norm_clust, "sgv-clust", 1); + res = sgv_pool_init(&pools->default_set.norm_clust, "sgv-clust", + sgv_full_clustering); if (res != 0) goto out_free_clust; - res = sgv_pool_init(&pools->default_set.dma, "sgv-dma", 0); + res = sgv_pool_init(&pools->default_set.dma, "sgv-dma", + sgv_no_clustering); if (res != 0) goto out_free_norm; diff --git a/scst/src/scst_mem.h b/scst/src/scst_mem.h index 426145e25..278970e86 100644 --- a/scst/src/scst_mem.h +++ b/scst/src/scst_mem.h @@ -73,7 +73,7 @@ struct sgv_pool_alloc_fns { }; struct sgv_pool { - unsigned int clustered; + enum sgv_clustering_types clustering_type; struct sgv_pool_alloc_fns alloc_fns; /* 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048 */ struct kmem_cache *caches[SGV_POOL_ELEMENTS]; @@ -139,7 +139,7 @@ struct scst_sgv_pools_manager { }; int sgv_pool_init(struct sgv_pool *pool, const char *name, - int clustered); + enum sgv_clustering_types clustering_type); void sgv_pool_deinit(struct sgv_pool *pool); static inline struct scatterlist *sgv_pool_sg(struct sgv_pool_obj *obj)