Index: scst/include/scst_sgv.h =================================================================== --- scst/include/scst_sgv.h (revision 821) +++ scst/include/scst_sgv.h (working copy) @@ -58,12 +58,17 @@ void sgv_pool_destroy(struct sgv_pool *p void sgv_pool_flush(struct sgv_pool *pool); void sgv_pool_set_allocator(struct sgv_pool *pool, - struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *), - void (*free_pages_fn)(struct scatterlist *, int, void *)); + struct page *(*alloc_pages_fn)(struct scatterlist *sg, + gfp_t gfp_mask, int alloc_order, void *priv), + void (*free_pages_fn)(struct scatterlist *sg, int sg_count, + int alloc_order, void *priv)); + +void sgv_pool_set_max_alloc_order(struct sgv_pool *pool, int max_alloc_order); struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size, gfp_t gfp_mask, int flags, int *count, - struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv); + struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv, + int max_sg_count); void sgv_pool_free(struct sgv_pool_obj *sgv, struct scst_mem_lim *mem_lim); void *sgv_get_priv(struct sgv_pool_obj *sgv); Index: scst/src/scst_mem.h =================================================================== --- scst/src/scst_mem.h (revision 821) +++ scst/src/scst_mem.h (working copy) @@ -44,6 +44,7 @@ struct sgv_pool_obj { } recycle_entry; struct sgv_pool *owner_pool; + int alloc_order; int orig_sg; int orig_length; int sg_count; @@ -67,14 +68,21 @@ struct sgv_pool_cache_acc { struct sgv_pool_alloc_fns { struct page *(*alloc_pages_fn)(struct scatterlist *sg, gfp_t gfp_mask, - void *priv); + int alloc_order, void *priv); void (*free_pages_fn)(struct scatterlist *sg, int sg_count, - void *priv); + int alloc_order, void *priv); }; struct sgv_pool { enum sgv_clustering_types clustering_type; + + int max_alloc_order; + +#define SGV_POOL_MAX_SG_MISSED 1 + unsigned long sgv_pool_flags; + struct sgv_pool_alloc_fns alloc_fns; + /* 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048 */ struct kmem_cache *caches[SGV_POOL_ELEMENTS]; @@ -87,6 +95,7 @@ struct sgv_pool { /* SCST_MAX_NAME + few more bytes to match scst_user expectations */ char cache_names[SGV_POOL_ELEMENTS][SCST_MAX_NAME + 10]; char name[SCST_MAX_NAME + 10]; + struct list_head sgv_pool_list_entry; }; Index: scst/src/scst_lib.c =================================================================== --- scst/src/scst_lib.c (revision 821) +++ scst/src/scst_lib.c (working copy) @@ -2262,7 +2262,6 @@ int scst_alloc_space(struct scst_cmd *cm int atomic = scst_cmd_atomic(cmd); int flags; struct scst_tgt_dev *tgt_dev = cmd->tgt_dev; - static int ll; TRACE_ENTRY(); @@ -2273,40 +2272,25 @@ int scst_alloc_space(struct scst_cmd *cm flags |= SCST_POOL_ALLOC_NO_CACHED; cmd->sg = sgv_pool_alloc(tgt_dev->pool, cmd->bufflen, gfp_mask, flags, - &cmd->sg_cnt, &cmd->sgv, &cmd->dev->dev_mem_lim, NULL); + &cmd->sg_cnt, &cmd->sgv, + &cmd->dev->dev_mem_lim, NULL, + tgt_dev->max_sg_cnt); if (cmd->sg == NULL) goto out; - if (unlikely(cmd->sg_cnt > tgt_dev->max_sg_cnt)) { - if (ll < 10) { - PRINT_INFO("Unable to complete command due to " - "SG IO count limitation (requested %d, " - "available %d, tgt lim %d)", cmd->sg_cnt, - tgt_dev->max_sg_cnt, cmd->tgt->sg_tablesize); - ll++; - } - goto out_sg_free; - } + EXTRACHECKS_BUG_ON(cmd->sg_cnt > tgt_dev->max_sg_cnt); if (cmd->data_direction != SCST_DATA_BIDI) goto success; cmd->in_sg = sgv_pool_alloc(tgt_dev->pool, cmd->in_bufflen, gfp_mask, - flags, &cmd->in_sg_cnt, &cmd->in_sgv, - &cmd->dev->dev_mem_lim, NULL); + flags, &cmd->in_sg_cnt, &cmd->in_sgv, + &cmd->dev->dev_mem_lim, NULL, + tgt_dev->max_sg_cnt); if (cmd->in_sg == NULL) goto out_sg_free; - if (unlikely(cmd->in_sg_cnt > tgt_dev->max_sg_cnt)) { - if (ll < 10) { - PRINT_INFO("Unable to complete command due to " - "SG IO count limitation (IN buffer, requested " - "%d, available %d, tgt lim %d)", cmd->in_sg_cnt, - tgt_dev->max_sg_cnt, cmd->tgt->sg_tablesize); - ll++; - } - goto out_in_sg_free; - } + EXTRACHECKS_BUG_ON(cmd->in_sg_cnt > tgt_dev->max_sg_cnt); success: res = 0; @@ -2315,12 +2299,6 @@ out: TRACE_EXIT(); return res; -out_in_sg_free: - sgv_pool_free(cmd->in_sgv, &cmd->dev->dev_mem_lim); - cmd->in_sgv = NULL; - cmd->in_sg = NULL; - cmd->in_sg_cnt = 0; - out_sg_free: sgv_pool_free(cmd->sgv, &cmd->dev->dev_mem_lim); cmd->sgv = NULL; Index: scst/src/scst_targ.c =================================================================== --- scst/src/scst_targ.c (revision 821) +++ scst/src/scst_targ.c (working copy) @@ -2066,7 +2066,7 @@ static int scst_do_real_exec(struct scst res = SCST_EXEC_NEED_THREAD; goto out_restore; } else { - PRINT_ERROR("scst_exec_req() failed: %d", res); + PRINT_ERROR("scst_exec_req() failed: %d", rc); goto out_error; } } Index: scst/src/scst_mem.c =================================================================== --- scst/src/scst_mem.c (revision 821) +++ scst/src/scst_mem.c (working copy) @@ -168,52 +168,39 @@ out: } static void scst_free_sys_sg_entries(struct scatterlist *sg, int sg_count, - void *priv) + int alloc_order, void *priv) { int i; + const int num_pages = 1 << alloc_order; - TRACE_MEM("sg=%p, sg_count=%d", sg, sg_count); + TRACE_MEM("sg=%p, sg_count=%d, alloc_order=%d", sg, sg_count, alloc_order); for (i = 0; i < sg_count; i++) { struct page *p = sg_page(&sg[i]); int len = sg[i].length; - int pages = - (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0); + int pages = (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0); TRACE_MEM("page %lx, len %d, pages %d", (unsigned long)p, len, pages); while (pages > 0) { - int order = 0; - -/* - * __free_pages() doesn't like freeing pages with not that order with - * which they were allocated, so disable this small optimization. - */ -#if 0 - if (len > 0) { - while (((1 << order) << PAGE_SHIFT) < len) - order++; - len = 0; - } -#endif TRACE_MEM("free_pages(): order %d, page %lx", - order, (unsigned long)p); + alloc_order, (unsigned long)p); - __free_pages(p, order); + __free_pages(p, alloc_order); - pages -= 1 << order; - p += 1 << order; + pages -= num_pages; + p += num_pages; } } } static struct page *scst_alloc_sys_pages(struct scatterlist *sg, - gfp_t gfp_mask, void *priv) + gfp_t gfp_mask, int alloc_order, void *priv) { - struct page *page = alloc_pages(gfp_mask, 0); + struct page *page = alloc_pages(gfp_mask, alloc_order); - sg_set_page(sg, page, PAGE_SIZE, 0); + sg_set_page(sg, page, PAGE_SIZE << alloc_order, 0); TRACE_MEM("page=%p, sg=%p, priv=%p", page, sg, priv); if (page == NULL) { TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of " @@ -225,13 +212,15 @@ static struct page *scst_alloc_sys_pages static int scst_alloc_sg_entries(struct scatterlist *sg, int pages, gfp_t gfp_mask, enum sgv_clustering_types clustering_type, struct trans_tbl_ent *trans_tbl, - const struct sgv_pool_alloc_fns *alloc_fns, void *priv) + const struct sgv_pool_alloc_fns *alloc_fns, int alloc_order, + void *priv) { int sg_count = 0; int pg, i, j; int merged = -1; - TRACE_MEM("pages=%d, clustering_type=%d", pages, clustering_type); + TRACE_MEM("pages=%d, clustering_type=%d, alloc_order=%d", pages, + clustering_type, alloc_order); #if 0 gfp_mask |= __GFP_COLD; @@ -240,7 +229,7 @@ static int scst_alloc_sg_entries(struct gfp_mask |= __GFP_ZERO; #endif - for (pg = 0; pg < pages; pg++) { + for (pg = 0; pg < pages; pg += 1 << alloc_order) { void *rc; #ifdef CONFIG_SCST_DEBUG_OOM if (((gfp_mask & __GFP_NOFAIL) != __GFP_NOFAIL) && @@ -249,7 +238,7 @@ static int scst_alloc_sg_entries(struct else #endif rc = alloc_fns->alloc_pages_fn(&sg[sg_count], gfp_mask, - priv); + alloc_order, priv); if (rc == NULL) goto out_no_mem; @@ -263,8 +252,8 @@ static int scst_alloc_sg_entries(struct if (merged == -1) sg_count++; - TRACE_MEM("pg=%d, merged=%d, sg_count=%d", pg, merged, - sg_count); + TRACE_MEM("pg=%d, merged=%d, sg_count=%d", + pg, merged, sg_count); } if ((clustering_type != sgv_no_clustering) && (trans_tbl != NULL)) { @@ -285,7 +274,7 @@ out: return sg_count; out_no_mem: - alloc_fns->free_pages_fn(sg, sg_count, priv); + alloc_fns->free_pages_fn(sg, sg_count, alloc_order, priv); sg_count = 0; goto out; } @@ -346,9 +335,11 @@ out_free: static void sgv_dtor_and_free(struct sgv_pool_obj *obj) { + TRACE_MEM("Destroing obj %p", obj); + if (obj->sg_count != 0) { obj->owner_pool->alloc_fns.free_pages_fn(obj->sg_entries, - obj->sg_count, obj->allocator_priv); + obj->sg_count, obj->alloc_order, obj->allocator_priv); } if (obj->sg_entries != obj->sg_entries_data) { if (obj->trans_tbl != @@ -364,53 +355,59 @@ static void sgv_dtor_and_free(struct sgv return; } -static struct sgv_pool_obj *sgv_pool_cached_get(struct sgv_pool *pool, - int order, gfp_t gfp_mask) +static struct sgv_pool_obj *sgv_cached_obj_get(struct sgv_pool *pool, + int order, gfp_t gfp_mask, int max_sg_count) { struct sgv_pool_obj *obj; int pages = 1 << order; spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); - if (likely(!list_empty(&pool->recycling_lists[order]))) { - obj = list_entry(pool->recycling_lists[order].next, - struct sgv_pool_obj, - recycle_entry.recycling_list_entry); - list_del(&obj->recycle_entry.sorted_recycling_list_entry); - list_del(&obj->recycle_entry.recycling_list_entry); + if (!list_empty(&pool->recycling_lists[order])) { + list_for_each_entry(obj, &pool->recycling_lists[order], + recycle_entry.recycling_list_entry) { - sgv_pools_mgr.mgr.throttle.inactive_pages_total -= pages; - sgv_pools_mgr.mgr.throttle.active_pages_total += pages; + TRACE_MEM("obj %p, sg_count %d (max %d)", obj, + obj->sg_count, max_sg_count); - spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); + if (unlikely(obj->sg_count > max_sg_count)) + continue; - EXTRACHECKS_BUG_ON(obj->order_or_pages != order); - goto out; - } + list_del(&obj->recycle_entry.sorted_recycling_list_entry); + list_del(&obj->recycle_entry.recycling_list_entry); - pool->acc.cached_entries++; - pool->acc.cached_pages += pages; + sgv_pools_mgr.mgr.throttle.inactive_pages_total -= pages; + sgv_pools_mgr.mgr.throttle.active_pages_total += pages; - spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); + spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); - obj = kmem_cache_alloc(pool->caches[order], - gfp_mask & ~(__GFP_HIGHMEM|GFP_DMA)); - if (likely(obj)) { - memset(obj, 0, sizeof(*obj)); - obj->order_or_pages = order; - obj->owner_pool = pool; + EXTRACHECKS_BUG_ON(obj->order_or_pages != order); + break; + } } else { - spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); - pool->acc.cached_entries--; - pool->acc.cached_pages -= pages; + pool->acc.cached_entries++; + pool->acc.cached_pages += pages; + spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); + + obj = kmem_cache_alloc(pool->caches[order], + gfp_mask & ~(__GFP_HIGHMEM|GFP_DMA)); + if (likely(obj != NULL)) { + memset(obj, 0, sizeof(*obj)); + obj->order_or_pages = order; + obj->owner_pool = pool; + } else { + spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); + pool->acc.cached_entries--; + pool->acc.cached_pages -= pages; + spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); + } } -out: return obj; } -static void sgv_pool_cached_put(struct sgv_pool_obj *sgv) +static void sgv_cached_obj_put(struct sgv_pool_obj *sgv) { struct sgv_pool *owner = sgv->owner_pool; struct list_head *entry; @@ -464,7 +461,7 @@ static void sgv_pool_cached_put(struct s } /* Must be called under pool_mgr_lock held */ -static void __sgv_pool_cached_purge(struct sgv_pool_obj *e) +static void __sgv_cached_purge_obj(struct sgv_pool_obj *e) { int pages = 1 << e->order_or_pages; @@ -478,13 +475,13 @@ static void __sgv_pool_cached_purge(stru } /* Must be called under pool_mgr_lock held */ -static int sgv_pool_cached_purge(struct sgv_pool_obj *e, int t, +static int sgv_cached_purge_obj(struct sgv_pool_obj *e, int t, unsigned long rt) { EXTRACHECKS_BUG_ON(t == 0); if (time_after(rt, (e->recycle_entry.time_stamp + t))) { - __sgv_pool_cached_purge(e); + __sgv_cached_purge_obj(e); return 0; } return 1; @@ -507,7 +504,7 @@ static int sgv_pool_oom_free_objs(int pg struct sgv_pool_obj, recycle_entry.sorted_recycling_list_entry); - __sgv_pool_cached_purge(e); + __sgv_cached_purge_obj(e); pgs -= 1 << e->order_or_pages; spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); @@ -602,12 +599,13 @@ static void scst_uncheck_allowed_mem(str struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size, gfp_t gfp_mask, int flags, int *count, - struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv) + struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv, + int max_sg_count) { struct sgv_pool_obj *obj; int order, pages, cnt; struct scatterlist *res = NULL; - int pages_to_alloc; + int pages_to_alloc, alloc_order; struct kmem_cache *cache; int no_cached = flags & SCST_POOL_ALLOC_NO_CACHED; bool allowed_mem_checked = false, hiwmk_checked = false; @@ -643,7 +641,10 @@ struct scatterlist *sgv_pool_alloc(struc if (unlikely(sgv_pool_hiwmk_check(pages_to_alloc) != 0)) goto out_fail_free_sg_entries; hiwmk_checked = true; - } else if ((order < SGV_POOL_ELEMENTS) && !no_cached) { + goto alloc; + } + + if ((order < SGV_POOL_ELEMENTS) && !no_cached) { pages_to_alloc = (1 << order); cache = pool->caches[order]; @@ -651,7 +652,7 @@ struct scatterlist *sgv_pool_alloc(struc goto out_fail; allowed_mem_checked = true; - obj = sgv_pool_cached_get(pool, order, gfp_mask); + obj = sgv_cached_obj_get(pool, order, gfp_mask, max_sg_count); if (unlikely(obj == NULL)) { TRACE(TRACE_OUT_OF_MEM, "Allocation of " "sgv_pool_obj failed (size %d)", size); @@ -659,7 +660,8 @@ struct scatterlist *sgv_pool_alloc(struc } if (obj->sg_count != 0) { - TRACE_MEM("Cached sgv_obj %p", obj); + TRACE_MEM("Cached sgv_obj %p (order_or_pages %d, " + "order %d)", obj, obj->order_or_pages, order); EXTRACHECKS_BUG_ON(obj->order_or_pages != order); atomic_inc(&pool->cache_acc[order].hit_alloc); goto success; @@ -738,15 +740,73 @@ struct scatterlist *sgv_pool_alloc(struc TRACE_MEM("Big or no_cached sgv_obj %p (size %d)", obj, sz); } - obj->sg_count = scst_alloc_sg_entries(obj->sg_entries, - pages_to_alloc, gfp_mask, pool->clustering_type, - obj->trans_tbl, &pool->alloc_fns, priv); - if (unlikely(obj->sg_count <= 0)) { - obj->sg_count = 0; - if ((flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL) && cache) - goto out_return1; - else +alloc: + /* + * Allocate the scatter gather entries. Since the memory we + * request may fit in too many entries, we try to start with + * an order big enough. That will save some useless + * allocations. + */ + alloc_order = 0; + if (pages_to_alloc > max_sg_count) { + if (test_bit(SGV_POOL_MAX_SG_MISSED, &pool->sgv_pool_flags)) { + int tmp = pages_to_alloc; + do { + tmp >>= 1; + alloc_order++; + } while (tmp > max_sg_count); + + /* Let's hope for clustering */ + if ((cache != NULL) && + (pool->clustering_type != sgv_no_clustering)) + alloc_order--; + + if (alloc_order > pool->max_alloc_order) + alloc_order = pool->max_alloc_order; + } + } + + TRACE_MEM("Starting alloc order %d", alloc_order); + + while (1) { + obj->sg_count = scst_alloc_sg_entries(obj->sg_entries, + pages_to_alloc, gfp_mask, pool->clustering_type, + obj->trans_tbl, &pool->alloc_fns, alloc_order, priv); + if (unlikely(obj->sg_count <= 0)) { + obj->sg_count = 0; + if ((cache != NULL) && + (flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL)) + goto out_return1; + else + goto out_fail_free_sg_entries; + } + + obj->alloc_order = alloc_order; + + if (likely(obj->sg_count <= max_sg_count)) + break; + + set_bit(SGV_POOL_MAX_SG_MISSED, &pool->sgv_pool_flags); + + obj->owner_pool->alloc_fns.free_pages_fn(obj->sg_entries, + obj->sg_count, obj->alloc_order, obj->allocator_priv); + alloc_order++; + + TRACE_MEM("Too many sg_count %d (max %d), trying next order %d " + "(max order %d)", obj->sg_count, max_sg_count, + alloc_order, pool->max_alloc_order); + + if (alloc_order > pool->max_alloc_order) { + static int ll; + if (ll < 20) { + PRINT_INFO("Unable to make %d segments " + "allocation, max order %d exceeded " + "(size %d)", max_sg_count, + alloc_order-1, size); + ll++; + } goto out_fail_free_sg_entries; + } } if (cache) { @@ -773,7 +833,7 @@ success: if (sgv_pool_clustered(pool)) cnt = obj->trans_tbl[pages-1].sg_num; else - cnt = pages; + cnt = obj->sg_count; sg = cnt-1; obj->orig_sg = sg; obj->orig_length = obj->sg_entries[sg].length; @@ -831,9 +891,13 @@ out_fail_free_sg_entries: } out_fail_free: - if (cache) - sgv_pool_cached_put(obj); - else + if (cache) { + spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); + pool->acc.cached_entries--; + pool->acc.cached_pages -= pages_to_alloc; + spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); + kmem_cache_free(pool->caches[obj->order_or_pages], obj); + } else kfree(obj); out_fail: @@ -868,10 +932,10 @@ void sgv_pool_free(struct sgv_pool_obj * if (sgv->order_or_pages >= 0) { sgv->sg_entries[sgv->orig_sg].length = sgv->orig_length; pages = (sgv->sg_count != 0) ? 1 << sgv->order_or_pages : 0; - sgv_pool_cached_put(sgv); + sgv_cached_obj_put(sgv); } else { sgv->owner_pool->alloc_fns.free_pages_fn(sgv->sg_entries, - sgv->sg_count, sgv->allocator_priv); + sgv->sg_count, sgv->alloc_order, sgv->allocator_priv); pages = (sgv->sg_count != 0) ? -sgv->order_or_pages : 0; kfree(sgv); sgv_pool_hiwmk_uncheck(pages); @@ -924,7 +988,7 @@ struct scatterlist *scst_alloc(int size, * So, always don't use clustering. */ *count = scst_alloc_sg_entries(res, pages, gfp_mask, sgv_no_clustering, - NULL, &sys_alloc_fns, NULL); + NULL, &sys_alloc_fns, 0, NULL); if (*count <= 0) goto out_free; @@ -951,7 +1015,7 @@ void scst_free(struct scatterlist *sg, i sgv_pool_hiwmk_uncheck(count); - scst_free_sys_sg_entries(sg, count, NULL); + scst_free_sys_sg_entries(sg, count, 0, NULL); kfree(sg); return; } @@ -983,6 +1047,14 @@ int sgv_pool_init(struct sgv_pool *pool, atomic_set(&pool->acc.big_merged, 0); pool->clustering_type = clustering_type; + if (clustering_type == sgv_no_clustering) { + /* + * No clustering means always single page entries. For + * instance, iSCSI-SCST requires that. + */ + pool->max_alloc_order = 0; + } else + pool->max_alloc_order = 0x0FFF; /* infinity */ pool->alloc_fns.alloc_pages_fn = scst_alloc_sys_pages; pool->alloc_fns.free_pages_fn = scst_free_sys_sg_entries; @@ -1102,7 +1174,7 @@ void sgv_pool_flush(struct sgv_pool *poo struct sgv_pool_obj, recycle_entry.recycling_list_entry); - __sgv_pool_cached_purge(e); + __sgv_cached_purge_obj(e); spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); EXTRACHECKS_BUG_ON(e->owner_pool != pool); @@ -1141,8 +1213,10 @@ void sgv_pool_deinit(struct sgv_pool *po } void sgv_pool_set_allocator(struct sgv_pool *pool, - struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *), - void (*free_pages_fn)(struct scatterlist *, int, void *)) + struct page *(*alloc_pages_fn)(struct scatterlist *sg, + gfp_t gfp_mask, int alloc_order, void *priv), + void (*free_pages_fn)(struct scatterlist *sg, int sg_count, + int alloc_order, void *priv)) { pool->alloc_fns.alloc_pages_fn = alloc_pages_fn; pool->alloc_fns.free_pages_fn = free_pages_fn; @@ -1150,6 +1224,12 @@ void sgv_pool_set_allocator(struct sgv_p } EXPORT_SYMBOL(sgv_pool_set_allocator); +void sgv_pool_set_max_alloc_order(struct sgv_pool *pool, int max_alloc_order) +{ + pool->max_alloc_order = max_alloc_order; +} +EXPORT_SYMBOL(sgv_pool_set_max_alloc_order); + struct sgv_pool *sgv_pool_create(const char *name, enum sgv_clustering_types clustering_type) { @@ -1206,7 +1286,7 @@ static int sgv_pool_cached_shrinker(int struct sgv_pool_obj, recycle_entry.sorted_recycling_list_entry); - if (sgv_pool_cached_purge(e, SHRINK_TIME_AFTER, rt) == 0) { + if (sgv_cached_purge_obj(e, SHRINK_TIME_AFTER, rt) == 0) { nr -= 1 << e->order_or_pages; spin_unlock_bh( &sgv_pools_mgr.mgr.pool_mgr_lock); @@ -1228,7 +1308,7 @@ static int sgv_pool_cached_shrinker(int return nr; } -static void sgv_pool_cached_pitbool(void *p) +static void sgv_purge_work(void *p) { u32 total_pages; struct sgv_pool_obj *e; @@ -1245,7 +1325,7 @@ static void sgv_pool_cached_pitbool(void struct sgv_pool_obj, recycle_entry.sorted_recycling_list_entry); - if (sgv_pool_cached_purge(e, PURGE_TIME_AFTER, rt) == 0) { + if (sgv_cached_purge_obj(e, PURGE_TIME_AFTER, rt) == 0) { spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); sgv_dtor_and_free(e); spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock); @@ -1304,9 +1384,9 @@ int scst_sgv_pools_init(unsigned long me #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20)) INIT_DELAYED_WORK(&pools->mgr.apit_pool, - (void (*)(struct work_struct *))sgv_pool_cached_pitbool); + (void (*)(struct work_struct *))sgv_purge_work); #else - INIT_WORK(&pools->mgr.apit_pool, sgv_pool_cached_pitbool, NULL); + INIT_WORK(&pools->mgr.apit_pool, sgv_purge_work, NULL); #endif #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23)) Index: scst/src/dev_handlers/scst_user.c =================================================================== --- scst/src/dev_handlers/scst_user.c (revision 821) +++ scst/src/dev_handlers/scst_user.c (working copy) @@ -169,9 +169,9 @@ static int dev_user_disk_done(struct scs static int dev_user_tape_done(struct scst_cmd *cmd); static struct page *dev_user_alloc_pages(struct scatterlist *sg, - gfp_t gfp_mask, void *priv); + gfp_t gfp_mask, int alloc_order, void *priv); static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count, - void *priv); + int alloc_order, void *priv); static void dev_user_add_to_ready(struct scst_user_cmd *ucmd); @@ -366,7 +366,7 @@ static void dev_user_free_ucmd(struct sc } static struct page *dev_user_alloc_pages(struct scatterlist *sg, - gfp_t gfp_mask, void *priv) + gfp_t gfp_mask, int alloc_order, void *priv) { struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv; int offset = 0; @@ -375,8 +375,10 @@ static struct page *dev_user_alloc_pages /* *sg supposed to be zeroed */ - TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd, - ucmd->ubuff, ucmd->cur_data_page); + TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d, alloc_order %d", + ucmd, ucmd->ubuff, ucmd->cur_data_page, alloc_order); + + EXTRACHECKS_BUG_ON(alloc_order != 0); if (ucmd->cur_data_page == 0) { TRACE_MEM("ucmd->first_page_offset %d", @@ -469,7 +471,7 @@ static void __dev_user_free_sg_entries(s } static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count, - void *priv) + int alloc_order, void *priv) { struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv; @@ -558,7 +560,8 @@ static int dev_user_alloc_sg(struct scst ucmd->buff_cached = cached_buff; cmd->sg = sgv_pool_alloc(pool, bufflen, gfp_mask, flags, &cmd->sg_cnt, - &ucmd->sgv, &dev->udev_mem_lim, ucmd); + &ucmd->sgv, &dev->udev_mem_lim, ucmd, + cmd->tgt_dev->max_sg_cnt); if (cmd->sg != NULL) { struct scst_user_cmd *buf_ucmd = (struct scst_user_cmd *)sgv_get_priv(ucmd->sgv); @@ -589,21 +592,6 @@ static int dev_user_alloc_sg(struct scst TRACE_MEM("cmd %p, in_sg %p, in_sg_cnt %d, sg_cnt %d", cmd, cmd->in_sg, cmd->in_sg_cnt, cmd->sg_cnt); } - - if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) { - static int ll; - if (ll < 10) { - PRINT_INFO("Unable to complete command due to " - "SG IO count limitation (requested %d, " - "available %d, tgt lim %d)", - cmd->sg_cnt, cmd->tgt_dev->max_sg_cnt, - cmd->tgt->sg_tablesize); - ll++; - } - cmd->sg = NULL; - /* sgv will be freed in dev_user_free_sgv() */ - res = -1; - } } else { TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, " "sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h, @@ -2713,6 +2701,7 @@ static int dev_user_register_dev(struct goto out_free_dev; sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages, dev_user_free_sg_entries); + sgv_pool_set_max_alloc_order(dev->pool, 0); scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s-clust", dev->name); @@ -2722,6 +2711,7 @@ static int dev_user_register_dev(struct goto out_free0; sgv_pool_set_allocator(dev->pool_clust, dev_user_alloc_pages, dev_user_free_sg_entries); + sgv_pool_set_max_alloc_order(dev->pool_clust, 0); scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s", dev->name);