Clustering of scst_user pages added

git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@663 d57e44dd-8a1f-0410-8b47-8ef2f437770f

scst/include/scst.h

@@ -364,7 +364,7 @@ enum scst_exec_context {
 #define SCST_CMD_CAN_BE_DESTROYED	3
 
 /*************************************************************
- ** Tgt_dev's flags (tgt_dev_flags)
+ ** Tgt_dev's async. flags (tgt_dev_flags)
  *************************************************************/
 
 /* Set if tgt_dev has Unit Attention sense */
@@ -381,6 +381,8 @@ enum scst_exec_context {
 #define SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC	9
 #define SCST_TGT_DEV_AFTER_EXEC_ATOMIC		10
 
+#define SCST_TGT_DEV_CLUST_POOL			11
+
 #ifdef CONFIG_SCST_DEBUG_TM
 #define SCST_TGT_DEV_UNDER_TM_DBG		20
 #endif
@@ -1501,7 +1503,7 @@ struct scst_tgt_dev {
 
 	/*
 	 * Set if the prev cmd was ORDERED. Size must allow unprotected
-	 * modifications
+	 * modifications independant to the neighbour fields. 
 	 */
 	unsigned long prev_cmd_ordered;
 

scst/include/scst_sgv.h

@@ -43,7 +43,15 @@ struct scst_mem_lim {
 	int max_allowed_pages;
 };
 
-struct sgv_pool *sgv_pool_create(const char *name, int clustered);
+/* Types of clustering */
+enum sgv_clustering_types {
+	sgv_no_clustering = 0,
+	sgv_tail_clustering,
+	sgv_full_clustering,
+};
+
+struct sgv_pool *sgv_pool_create(const char *name,
+	enum sgv_clustering_types clustered);
 void sgv_pool_destroy(struct sgv_pool *pool);
 
 void sgv_pool_set_allocator(struct sgv_pool *pool,

scst/src/dev_handlers/scst_user.c

@@ -82,6 +82,7 @@ struct scst_user_dev {
 
 	struct scst_mem_lim udev_mem_lim;
 	struct sgv_pool *pool;
+	struct sgv_pool *pool_clust;
 
 	uint8_t parse_type;
 	uint8_t on_free_cmd_type;
@@ -278,12 +279,6 @@ static inline int calc_num_pg(unsigned long buf, int len)
 	return (len >> PAGE_SHIFT) + ((len & ~PAGE_MASK) != 0);
 }
 
-static inline int is_need_offs_page(unsigned long buf, int len)
-{
-	return ((buf & ~PAGE_MASK) != 0) &&
-		((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
-}
-
 static void __dev_user_not_reg(void)
 {
 	PRINT_ERROR("%s", "Device not registered");
@@ -497,6 +492,12 @@ static inline int is_buff_cached(struct scst_user_cmd *ucmd)
 		return 0;
 }
 
+static inline int is_need_offs_page(unsigned long buf, int len)
+{
+	return ((buf & ~PAGE_MASK) != 0) &&
+		((buf & PAGE_MASK) != ((buf+len-1) & PAGE_MASK));
+}
+
 /*
  * Returns 0 for success, <0 for fatal failure, >0 - need pages.
  * Unmaps the buffer, if needed in case of error
@@ -510,6 +511,7 @@ static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
 	int flags = 0;
 	int bufflen = cmd->bufflen;
 	int last_len = 0;
+	struct sgv_pool *pool;
 
 	TRACE_ENTRY();
 
@@ -539,13 +541,21 @@ static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
 	}
 	ucmd->buff_cached = cached_buff;
 
-	cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
-			&cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
+	if (test_bit(SCST_TGT_DEV_CLUST_POOL, &cmd->tgt_dev->tgt_dev_flags))
+		pool = dev->pool_clust;
+	else
+		pool = dev->pool;
+
+	cmd->sg = sgv_pool_alloc((struct sgv_pool *)cmd->tgt_dev->dh_priv,
+			bufflen, gfp_mask, flags, &cmd->sg_cnt, &ucmd->sgv,
+			&dev->udev_mem_lim, ucmd);
 	if (cmd->sg != NULL) {
 		struct scst_user_cmd *buf_ucmd =
 			(struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
 
-		TRACE_MEM("Buf ucmd %p", buf_ucmd);
+		TRACE_MEM("Buf ucmd %p (cmd->sg_cnt %d, last seg len %d, "
+			"last_len %d, bufflen %d)", buf_ucmd, cmd->sg_cnt,
+			cmd->sg[cmd->sg_cnt-1].length, last_len, bufflen);
 
 		ucmd->ubuff = buf_ucmd->ubuff;
 		ucmd->buf_ucmd = buf_ucmd;
@@ -554,13 +564,13 @@ static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
 				   (ucmd != buf_ucmd));
 
 		if (last_len != 0) {
-			/* We don't use clustering, so the assignment is safe */
-			cmd->sg[cmd->sg_cnt-1].length = last_len;
+			cmd->sg[cmd->sg_cnt-1].length &= PAGE_MASK;
+			cmd->sg[cmd->sg_cnt-1].length += last_len;
 		}
 
 		TRACE_MEM("Buf alloced (ucmd %p, cached_buff %d, ubuff %lx, "
-			"last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
-			last_len, cmd->sg[cmd->sg_cnt-1].length);
+			"last seg len %d)", ucmd, cached_buff, ucmd->ubuff,
+			cmd->sg[cmd->sg_cnt-1].length);
 
 		if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
 			static int ll;
@@ -2311,6 +2321,16 @@ static int dev_user_attach_tgt(struct scst_tgt_dev *tgt_dev)
 
 	TRACE_ENTRY();
 
+	/*
+	 * We can't replace tgt_dev->pool, because it can be used to allocate
+	 * memory for SCST local commands, like REPORT LUNS, where there is no
+	 * corresponding ucmd. Otherwise we will crash in dev_user_alloc_sg().
+	 */
+ 	if (test_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags))
+		tgt_dev->dh_priv = dev->pool_clust;
+ 	else
+		tgt_dev->dh_priv = dev->pool;
+
 	ucmd = dev_user_alloc_ucmd(dev, GFP_KERNEL);
 	if (ucmd == NULL)
 		goto out_nomem;
@@ -2554,18 +2574,21 @@ static int dev_user_register_dev(struct file *file,
 
 	scst_init_mem_lim(&dev->udev_mem_lim);
 
-	/*
-	 * We don't use clustered pool, since it implies pages reordering,
-	 * which isn't possible with user space supplied buffers. Although
-	 * it's still possible to cluster pages by the tail of each other,
-	 * seems it doesn't worth the effort.
-	 */
-	dev->pool = sgv_pool_create(dev->name, 0);
+	dev->pool = sgv_pool_create(dev->name, sgv_no_clustering);
 	if (dev->pool == NULL)
-		goto out_put;
+		goto out_free_dev;
 	sgv_pool_set_allocator(dev->pool, dev_user_alloc_pages,
 		dev_user_free_sg_entries);
 
+	scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "%s-clust",
+		dev->name);
+	dev->pool_clust = sgv_pool_create(dev->devtype.name,
+				sgv_tail_clustering);
+	if (dev->pool_clust == NULL)
+		goto out_free0;
+	sgv_pool_set_allocator(dev->pool_clust, dev_user_alloc_pages,
+		dev_user_free_sg_entries);
+
 	scnprintf(dev->devtype.name, sizeof(dev->devtype.name), "dh-%s",
 		dev->name);
 	dev->devtype.type = dev_desc->type;
@@ -2645,9 +2668,13 @@ out_del_free:
 	spin_unlock(&dev_list_lock);
 
 out_free:
+	sgv_pool_destroy(dev->pool_clust);
+
+out_free0:
 	sgv_pool_destroy(dev->pool);
+
+out_free_dev:
 	kfree(dev);
-	goto out_put;
 
 out_put:
 	module_put(THIS_MODULE);
@@ -2840,6 +2867,7 @@ static int dev_user_release(struct inode *inode, struct file *file)
 	scst_unregister_virtual_device(dev->virt_id);
 	scst_unregister_virtual_dev_driver(&dev->devtype);
 
+	sgv_pool_destroy(dev->pool_clust);
 	sgv_pool_destroy(dev->pool);
 
 	TRACE_DBG("Unregistering finished (dev %p)", dev);

scst/src/scst_mem.c

@@ -37,10 +37,16 @@
 
 static struct scst_sgv_pools_manager sgv_pools_mgr;
 
+static inline bool sgv_pool_clustered(const struct sgv_pool *pool)
+{
+	return (pool->clustering_type != sgv_no_clustering);
+}
+
 void scst_sgv_pool_use_norm(struct scst_tgt_dev *tgt_dev)
 {
 	tgt_dev->gfp_mask = __GFP_NOWARN;
 	tgt_dev->pool = &sgv_pools_mgr.default_set.norm;
+	clear_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags);
 }
 
 void scst_sgv_pool_use_norm_clust(struct scst_tgt_dev *tgt_dev)
@@ -48,6 +54,7 @@ void scst_sgv_pool_use_norm_clust(struct scst_tgt_dev *tgt_dev)
 	TRACE_MEM("%s", "Use clustering");
 	tgt_dev->gfp_mask = __GFP_NOWARN;
 	tgt_dev->pool = &sgv_pools_mgr.default_set.norm_clust;
+	set_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags);
 }
 
 void scst_sgv_pool_use_dma(struct scst_tgt_dev *tgt_dev)
@@ -55,9 +62,10 @@ void scst_sgv_pool_use_dma(struct scst_tgt_dev *tgt_dev)
 	TRACE_MEM("%s", "Use ISA DMA memory");
 	tgt_dev->gfp_mask = __GFP_NOWARN | GFP_DMA;
 	tgt_dev->pool = &sgv_pools_mgr.default_set.dma;
+	clear_bit(SCST_TGT_DEV_CLUST_POOL, &tgt_dev->tgt_dev_flags);
 }
 
-static int scst_check_clustering(struct scatterlist *sg, int cur, int hint)
+static int sgv_check_full_clustering(struct scatterlist *sg, int cur, int hint)
 {
 	int res = -1;
 	int i = hint;
@@ -65,7 +73,8 @@ static int scst_check_clustering(struct scatterlist *sg, int cur, int hint)
 	int len_cur = sg[cur].length;
 	unsigned long pfn_cur_next = pfn_cur + (len_cur >> PAGE_SHIFT);
 	int full_page_cur = (len_cur & (PAGE_SIZE - 1)) == 0;
-	unsigned long pfn, pfn_next, full_page;
+	unsigned long pfn, pfn_next;
+	bool full_page;
 
 #if 0
 	TRACE_MEM("pfn_cur %ld, pfn_cur_next %ld, len_cur %d, full_page_cur %d",
@@ -117,6 +126,47 @@ out_head:
 	goto out;
 }
 
+static int sgv_check_tail_clustering(struct scatterlist *sg, int cur, int hint)
+{
+	int res = -1;
+	unsigned long pfn_cur = page_to_pfn(sg_page(&sg[cur]));
+	int len_cur = sg[cur].length;
+	int prev;
+	unsigned long pfn_prev;
+	bool full_page;
+
+#ifdef SCST_HIGHMEM
+	if (page >= highmem_start_page) {
+		TRACE_MEM("%s", "HIGHMEM page allocated, no clustering")
+		goto out;
+	}
+#endif
+
+#if 0
+	TRACE_MEM("pfn_cur %ld, pfn_cur_next %ld, len_cur %d, full_page_cur %d",
+		pfn_cur, pfn_cur_next, len_cur, full_page_cur);
+#endif
+
+	if (cur == 0)
+		goto out;
+
+	prev = cur - 1;
+	pfn_prev = page_to_pfn(sg_page(&sg[prev])) +
+			(sg[prev].length >> PAGE_SHIFT);
+	full_page = (sg[prev].length & (PAGE_SIZE - 1)) == 0;
+
+	if ((pfn_prev == pfn_cur) && full_page) {
+		TRACE_MEM("SG segment %d will be tail merged with segment %d",
+			cur, prev);
+		sg[prev].length += len_cur;
+		sg_clear(&sg[cur]);
+		res = prev;
+	}
+
+out:
+	return res;
+}
+
 static void scst_free_sys_sg_entries(struct scatterlist *sg, int sg_count,
 	void *priv)
 {
@@ -173,14 +223,15 @@ static struct page *scst_alloc_sys_pages(struct scatterlist *sg,
 }
 
 static int scst_alloc_sg_entries(struct scatterlist *sg, int pages,
-	gfp_t gfp_mask, int clustered, struct trans_tbl_ent *trans_tbl,
+	gfp_t gfp_mask, enum sgv_clustering_types clustering_type,
+	struct trans_tbl_ent *trans_tbl,
 	const struct sgv_pool_alloc_fns *alloc_fns, void *priv)
 {
 	int sg_count = 0;
 	int pg, i, j;
 	int merged = -1;
 
-	TRACE_MEM("pages=%d, clustered=%d", pages, clustered);
+	TRACE_MEM("pages=%d, clustering_type=%d", pages, clustering_type);
 
 #if 0
 	gfp_mask |= __GFP_COLD;
@@ -201,17 +252,22 @@ static int scst_alloc_sg_entries(struct scatterlist *sg, int pages,
 				priv);
 		if (rc == NULL)
 			goto out_no_mem;
-		if (clustered) {
-			merged = scst_check_clustering(sg, sg_count, merged);
-			if (merged == -1)
-				sg_count++;
-		} else
+
+		if (clustering_type == sgv_full_clustering)
+			merged = sgv_check_full_clustering(sg, sg_count, merged);
+		else if (clustering_type == sgv_tail_clustering)
+			merged = sgv_check_tail_clustering(sg, sg_count, merged);
+		else
+			merged = -1;
+
+		if (merged == -1)
 			sg_count++;
+
 		TRACE_MEM("pg=%d, merged=%d, sg_count=%d", pg, merged,
 			sg_count);
 	}
 
-	if (clustered && (trans_tbl != NULL)) {
+	if ((clustering_type != sgv_no_clustering) && (trans_tbl != NULL)) {
 		pg = 0;
 		for (i = 0; i < pages; i++) {
 			int n = (sg[i].length >> PAGE_SHIFT) +
@@ -254,7 +310,7 @@ static int sgv_alloc_arrays(struct sgv_pool_obj *obj,
 
 	sg_init_table(obj->sg_entries, pages_to_alloc);
 
-	if (obj->owner_pool->clustered) {
+	if (sgv_pool_clustered(obj->owner_pool)) {
 		if (order <= sgv_pools_mgr.sgv_max_trans_order) {
 			obj->trans_tbl =
 				(struct trans_tbl_ent *)obj->sg_entries_data;
@@ -369,7 +425,7 @@ static void sgv_pool_cached_put(struct sgv_pool_obj *sgv)
 	TRACE_MEM("sgv %p, order %d, sg_count %d", sgv, sgv->order_or_pages,
 		sgv->sg_count);
 
-	if (owner->clustered) {
+	if (sgv_pool_clustered(owner)) {
 		/* Make objects with less entries more preferred */
 		__list_for_each(entry, list) {
 			struct sgv_pool_obj *tmp = list_entry(entry,
@@ -620,7 +676,7 @@ struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size,
 			obj->sg_entries = obj->sg_entries_data;
 			sg_init_table(obj->sg_entries, pages_to_alloc);
 			TRACE_MEM("sg_entries %p", obj->sg_entries);
-			if (pool->clustered) {
+			if (sgv_pool_clustered(pool)) {
 				obj->trans_tbl = (struct trans_tbl_ent *)
 					(obj->sg_entries + pages_to_alloc);
 				TRACE_MEM("trans_tbl %p", obj->trans_tbl);
@@ -683,8 +739,8 @@ struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size,
 	}
 
 	obj->sg_count = scst_alloc_sg_entries(obj->sg_entries,
-		pages_to_alloc, gfp_mask, pool->clustered, obj->trans_tbl,
-		&pool->alloc_fns, priv);
+		pages_to_alloc, gfp_mask, pool->clustering_type,
+		obj->trans_tbl, &pool->alloc_fns, priv);
 	if (unlikely(obj->sg_count <= 0)) {
 		obj->sg_count = 0;
 		if ((flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL) && cache)
@@ -714,14 +770,14 @@ success:
 	if (cache) {
 		int sg;
 		atomic_inc(&pool->cache_acc[order].total_alloc);
-		if (pool->clustered)
+		if (sgv_pool_clustered(pool))
 			cnt = obj->trans_tbl[pages-1].sg_num;
 		else
 			cnt = pages;
 		sg = cnt-1;
 		obj->orig_sg = sg;
 		obj->orig_length = obj->sg_entries[sg].length;
-		if (pool->clustered) {
+		if (sgv_pool_clustered(pool)) {
 			obj->sg_entries[sg].length =
 				(pages - obj->trans_tbl[sg].pg_count) << PAGE_SHIFT;
 		}
@@ -860,8 +916,8 @@ struct scatterlist *scst_alloc(int size, gfp_t gfp_mask, int *count)
 	 * scst_free() to figure out how many pages are in the SG vector.
 	 * So, always don't use clustering.
 	 */
-	*count = scst_alloc_sg_entries(res, pages, gfp_mask, 0, NULL,
-			&sys_alloc_fns, NULL);
+	*count = scst_alloc_sg_entries(res, pages, gfp_mask, sgv_no_clustering,
+			NULL, &sys_alloc_fns, NULL);
 	if (*count <= 0)
 		goto out_free;
 
@@ -901,7 +957,8 @@ static void sgv_pool_cached_init(struct sgv_pool *pool)
 		INIT_LIST_HEAD(&pool->recycling_lists[i]);
 }
 
-int sgv_pool_init(struct sgv_pool *pool, const char *name, int clustered)
+int sgv_pool_init(struct sgv_pool *pool, const char *name,
+	enum sgv_clustering_types clustering_type)
 {
 	int res = -ENOMEM;
 	int i;
@@ -918,12 +975,12 @@ int sgv_pool_init(struct sgv_pool *pool, const char *name, int clustered)
 	atomic_set(&pool->acc.other_merged, 0);
 	atomic_set(&pool->acc.big_merged, 0);
 
-	pool->clustered = clustered;
+	pool->clustering_type = clustering_type;
 	pool->alloc_fns.alloc_pages_fn = scst_alloc_sys_pages;
 	pool->alloc_fns.free_pages_fn = scst_free_sys_sg_entries;
 
-	TRACE_MEM("name %s, sizeof(*obj)=%zd, clustered=%d", name, sizeof(*obj),
-		clustered);
+	TRACE_MEM("name %s, sizeof(*obj)=%zd, clustering_type=%d", name,
+		sizeof(*obj), clustering_type);
 
 	strncpy(pool->name, name, sizeof(pool->name)-1);
 	pool->name[sizeof(pool->name)-1] = '\0';
@@ -938,14 +995,16 @@ int sgv_pool_init(struct sgv_pool *pool, const char *name, int clustered)
 		if (i <= sgv_pools_mgr.sgv_max_local_order) {
 			size = sizeof(*obj) + (1 << i) *
 				(sizeof(obj->sg_entries[0]) +
-				 (clustered ? sizeof(obj->trans_tbl[0]) : 0));
+				 ((clustering_type != sgv_no_clustering) ?
+				 	sizeof(obj->trans_tbl[0]) : 0));
 		} else if (i <= sgv_pools_mgr.sgv_max_trans_order) {
 			/*
 			 * sgv ie sg_entries is allocated outside object, but
 			 * ttbl is still embedded.
 			 */
 			size = sizeof(*obj) + (1 << i) *
-				((clustered ? sizeof(obj->trans_tbl[0]) : 0));
+				(((clustering_type != sgv_no_clustering) ?
+					sizeof(obj->trans_tbl[0]) : 0));
 		} else {
 			size = sizeof(*obj);
 
@@ -1070,7 +1129,8 @@ void sgv_pool_set_allocator(struct sgv_pool *pool,
 }
 EXPORT_SYMBOL(sgv_pool_set_allocator);
 
-struct sgv_pool *sgv_pool_create(const char *name, int clustered)
+struct sgv_pool *sgv_pool_create(const char *name,
+	enum sgv_clustering_types clustering_type)
 {
 	struct sgv_pool *pool;
 	int rc;
@@ -1083,7 +1143,7 @@ struct sgv_pool *sgv_pool_create(const char *name, int clustered)
 		goto out;
 	}
 
-	rc = sgv_pool_init(pool, name, clustered);
+	rc = sgv_pool_init(pool, name, clustering_type);
 	if (rc != 0)
 		goto out_free;
 
@@ -1207,15 +1267,17 @@ int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark)
 	INIT_LIST_HEAD(&pools->mgr.sorted_recycling_list);
 	spin_lock_init(&pools->mgr.pool_mgr_lock);
 
-	res = sgv_pool_init(&pools->default_set.norm, "sgv", 0);
+	res = sgv_pool_init(&pools->default_set.norm, "sgv", sgv_no_clustering);
 	if (res != 0)
 		goto out;
 
-	res = sgv_pool_init(&pools->default_set.norm_clust, "sgv-clust", 1);
+	res = sgv_pool_init(&pools->default_set.norm_clust, "sgv-clust",
+		sgv_full_clustering);
 	if (res != 0)
 		goto out_free_clust;
 
-	res = sgv_pool_init(&pools->default_set.dma, "sgv-dma", 0);
+	res = sgv_pool_init(&pools->default_set.dma, "sgv-dma",
+		sgv_no_clustering);
 	if (res != 0)
 		goto out_free_norm;
 

scst/src/scst_mem.h

@@ -73,7 +73,7 @@ struct sgv_pool_alloc_fns {
 };
 
 struct sgv_pool {
-	unsigned int clustered;
+	enum sgv_clustering_types clustering_type;
 	struct sgv_pool_alloc_fns alloc_fns;
 	/* 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048 */
 	struct kmem_cache *caches[SGV_POOL_ELEMENTS];
@@ -139,7 +139,7 @@ struct scst_sgv_pools_manager {
 };
 
 int sgv_pool_init(struct sgv_pool *pool, const char *name,
-	int clustered);
+	enum sgv_clustering_types clustering_type);
 void sgv_pool_deinit(struct sgv_pool *pool);
 
 static inline struct scatterlist *sgv_pool_sg(struct sgv_pool_obj *obj)