Cache alignment review

git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@4809 d57e44dd-8a1f-0410-8b47-8ef2f437770f

iscsi-scst/kernel/conn.c

@@ -833,7 +833,7 @@ int conn_free(struct iscsi_conn *conn)
 
 	free_page((unsigned long)conn->read_iov);
 
-	kfree(conn);
+	kmem_cache_free(iscsi_conn_cache, conn);
 
 	if (list_empty(&session->conn_list)) {
 		sBUG_ON(session->sess_reinst_successor != NULL);
@@ -850,7 +850,7 @@ static int iscsi_conn_alloc(struct iscsi_session *session,
 	struct iscsi_conn *conn;
 	int res = 0;
 
-	conn = kzalloc(L1_CACHE_ALIGN(sizeof(*conn)), GFP_KERNEL);
+	conn = kmem_cache_zalloc(iscsi_conn_cache, GFP_KERNEL);
 	if (!conn) {
 		res = -ENOMEM;
 		goto out_err;
@@ -941,7 +941,7 @@ out_free_iov:
 	free_page((unsigned long)conn->read_iov);
 
 out_err_free_conn:
-	kfree(conn);
+	kmem_cache_free(iscsi_conn_cache, conn);
 
 out_err:
 	goto out;

iscsi-scst/kernel/iscsi.c

@@ -51,8 +51,13 @@ static struct kmem_cache *iscsi_cmnd_cache;
 static DEFINE_MUTEX(iscsi_threads_pool_mutex);
 static LIST_HEAD(iscsi_thread_pools_list);
 
+static struct kmem_cache *iscsi_thread_pool_cache;
+
 static struct iscsi_thread_pool *iscsi_main_thread_pool;
 
+struct kmem_cache *iscsi_conn_cache;
+struct kmem_cache *iscsi_sess_cache;
+
 static struct page *dummy_page;
 static struct scatterlist dummy_sg;
 
@@ -3892,6 +3897,48 @@ struct scst_tgt_template iscsi_template = {
 	.get_scsi_transport_version = iscsi_get_scsi_transport_version,
 };
 
+static void __iscsi_threads_pool_put(struct iscsi_thread_pool *p)
+{
+	struct iscsi_thread *t, *tt;
+
+	TRACE_ENTRY();
+
+	p->thread_pool_ref--;
+	if (p->thread_pool_ref > 0) {
+		TRACE_DBG("iSCSI thread pool %p still has %d references)",
+			p, p->thread_pool_ref);
+		goto out;
+	}
+
+	TRACE_DBG("Freeing iSCSI thread pool %p", p);
+
+	list_for_each_entry_safe(t, tt, &p->threads_list, threads_list_entry) {
+		kthread_stop(t->thr);
+		list_del(&t->threads_list_entry);
+		kfree(t);
+	}
+
+	list_del(&p->thread_pools_list_entry);
+
+	kmem_cache_free(iscsi_thread_pool_cache, p);
+
+out:
+	TRACE_EXIT();
+	return;
+}
+
+void iscsi_threads_pool_put(struct iscsi_thread_pool *p)
+{
+	TRACE_ENTRY();
+
+	mutex_lock(&iscsi_threads_pool_mutex);
+	__iscsi_threads_pool_put(p);
+	mutex_unlock(&iscsi_threads_pool_mutex);
+
+	TRACE_EXIT();
+	return;
+}
+
 int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
 	struct iscsi_thread_pool **out_pool)
 {
@@ -3918,7 +3965,7 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
 
 	TRACE_DBG("%s", "Creating new iSCSI thread pool");
 
-	p = kzalloc(L1_CACHE_ALIGN(sizeof(*p)), GFP_KERNEL);
+	p = kmem_cache_zalloc(iscsi_thread_pool_cache, GFP_KERNEL);
 	if (p == NULL) {
 		PRINT_ERROR("Unable to allocate iSCSI thread pool (size %zd)",
 			sizeof(*p));
@@ -3958,6 +4005,8 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
 			count++;
 	}
 
+	list_add_tail(&p->thread_pools_list_entry, &iscsi_thread_pools_list);
+
 	for (j = 0; j < 2; j++) {
 		int (*fn)(void *);
 		char name[25];
@@ -4004,7 +4053,6 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
 		}
 	}
 
-	list_add_tail(&p->thread_pools_list_entry, &iscsi_thread_pools_list);
 	res = 0;
 
 	TRACE_DBG("Created iSCSI thread pool %p", p);
@@ -4012,55 +4060,17 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
 out_unlock:
 	mutex_unlock(&iscsi_threads_pool_mutex);
 
-	if (out_pool != NULL)
-		*out_pool = p;
+	*out_pool = p;
 
 	TRACE_EXIT_RES(res);
 	return res;
 
 out_free:
-	list_for_each_entry_safe(t, tt, &p->threads_list, threads_list_entry) {
-		kthread_stop(t->thr);
-		list_del(&t->threads_list_entry);
-		kfree(t);
-	}
+	__iscsi_threads_pool_put(p);
+	p = NULL;
 	goto out_unlock;
 }
 
-void iscsi_threads_pool_put(struct iscsi_thread_pool *p)
-{
-	struct iscsi_thread *t, *tt;
-
-	TRACE_ENTRY();
-
-	mutex_lock(&iscsi_threads_pool_mutex);
-
-	p->thread_pool_ref--;
-	if (p->thread_pool_ref > 0) {
-		TRACE_DBG("iSCSI thread pool %p still has %d references)",
-			p, p->thread_pool_ref);
-		goto out_unlock;
-	}
-
-	TRACE_DBG("Freeing iSCSI thread pool %p", p);
-
-	list_for_each_entry_safe(t, tt, &p->threads_list, threads_list_entry) {
-		kthread_stop(t->thr);
-		list_del(&t->threads_list_entry);
-		kfree(t);
-	}
-
-	list_del(&p->thread_pools_list_entry);
-
-	kfree(p);
-
-out_unlock:
-	mutex_unlock(&iscsi_threads_pool_mutex);
-
-	TRACE_EXIT();
-	return;
-}
-
 static int __init iscsi_init(void)
 {
 	int err = 0;
@@ -4111,12 +4121,32 @@ static int __init iscsi_init(void)
 	if (err < 0)
 		goto out_reg;
 
-	iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
+	iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
 	if (!iscsi_cmnd_cache) {
 		err = -ENOMEM;
 		goto out_event;
 	}
 
+	iscsi_thread_pool_cache = KMEM_CACHE(iscsi_thread_pool,
+					SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
+	if (!iscsi_thread_pool_cache) {
+		err = -ENOMEM;
+		goto out_kmem_cmd;
+	}
+
+	iscsi_conn_cache = KMEM_CACHE(iscsi_conn, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
+	if (!iscsi_conn_cache) {
+		err = -ENOMEM;
+		goto out_kmem_tp;
+	}
+
+	iscsi_sess_cache = KMEM_CACHE(iscsi_session,
+				SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
+	if (!iscsi_sess_cache) {
+		err = -ENOMEM;
+		goto out_kmem_conn;
+	}
+
 	err = scst_register_target_template(&iscsi_template);
 	if (err < 0)
 		goto out_kmem;
@@ -4147,6 +4177,15 @@ out_reg_tmpl:
 	scst_unregister_target_template(&iscsi_template);
 
 out_kmem:
+	kmem_cache_destroy(iscsi_sess_cache);
+
+out_kmem_conn:
+	kmem_cache_destroy(iscsi_conn_cache);
+
+out_kmem_tp:
+	kmem_cache_destroy(iscsi_thread_pool_cache);
+
+out_kmem_cmd:
 	kmem_cache_destroy(iscsi_cmnd_cache);
 
 out_event:
@@ -4181,6 +4220,9 @@ static void __exit iscsi_exit(void)
 #endif
 	event_exit();
 
+	kmem_cache_destroy(iscsi_sess_cache);
+	kmem_cache_destroy(iscsi_conn_cache);
+	kmem_cache_destroy(iscsi_thread_pool_cache);
 	kmem_cache_destroy(iscsi_cmnd_cache);
 
 	scst_unregister_target_template(&iscsi_template);

iscsi-scst/kernel/iscsi.h

@@ -518,6 +518,9 @@ extern struct mutex target_mgmt_mutex;
 extern int ctr_open_state;
 extern const struct file_operations ctr_fops;
 
+extern struct kmem_cache *iscsi_conn_cache;
+extern struct kmem_cache *iscsi_sess_cache;
+
 /* iscsi.c */
 extern struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *,
 	struct iscsi_cmnd *parent);

iscsi-scst/kernel/session.c

@@ -43,7 +43,7 @@ static int iscsi_session_alloc(struct iscsi_target *target,
 	struct iscsi_session *session;
 	char *name = NULL;
 
-	session = kzalloc(L1_CACHE_ALIGN(sizeof(*session)), GFP_KERNEL);
+	session = kmem_cache_zalloc(iscsi_sess_cache, GFP_KERNEL);
 	if (!session)
 		return -ENOMEM;
 
@@ -114,7 +114,7 @@ err_unreg:
 err:
 	if (session) {
 		kfree(session->initiator_name);
-		kfree(session);
+		kmem_cache_free(iscsi_sess_cache, session);
 #ifdef CONFIG_SCST_PROC
 		kfree(name);
 #endif
@@ -277,7 +277,7 @@ out_err_unlock:
 static void __session_free(struct iscsi_session *session)
 {
 	kfree(session->initiator_name);
-	kfree(session);
+	kmem_cache_free(iscsi_sess_cache, session);
 }
 
 static void iscsi_unreg_sess_done(struct scst_session *scst_sess)

qla2x00t/qla2x00-target/qla2x00t.c

@@ -242,6 +242,8 @@ static struct scst_tgt_template tgt2x_template = {
 };
 
 static struct kmem_cache *q2t_cmd_cachep;
+static struct kmem_cache *q2t_sess_cachep;
+static struct kmem_cache *q2t_tgt_cachep;
 static struct kmem_cache *q2t_mgmt_cmd_cachep;
 static mempool_t *q2t_mgmt_cmd_mempool;
 
@@ -629,7 +631,7 @@ static void q2t_free_session_done(struct scst_session *scst_sess)
 
 	TRACE_MGMT_DBG("Unregistration of sess %p finished", sess);
 
-	kfree(sess);
+	kmem_cache_free(q2t_sess_cachep, sess);
 
 	if (tgt == NULL)
 		goto out;
@@ -1155,7 +1157,7 @@ static struct q2t_sess *q2t_create_sess(scsi_qla_host_t *ha, fc_port_t *fcport,
 
 	/* We are under tgt_mutex, so a new sess can't be added behind us */
 
-	sess = kzalloc(L1_CACHE_ALIGN(sizeof(*sess)), GFP_KERNEL);
+	sess = kmem_cache_zalloc(q2t_sess_cachep, GFP_KERNEL);
 	if (sess == NULL) {
 		PRINT_ERROR("qla2x00t(%ld): session allocation failed, "
 			"all commands from port %02x:%02x:%02x:%02x:"
@@ -1229,7 +1231,7 @@ out_free_sess_wwn:
 	/* go through */
 
 out_free_sess:
-	kfree(sess);
+	kmem_cache_free(q2t_sess_cachep, sess);
 	sess = NULL;
 	goto out;
 }
@@ -1442,7 +1444,7 @@ static int q2t_target_release(struct scst_tgt *scst_tgt)
 
 	TRACE_MGMT_DBG("Release of tgt %p finished", tgt);
 
-	kfree(tgt);
+	kmem_cache_free(q2t_tgt_cachep, tgt);
 
 	TRACE_EXIT();
 	return 0;
@@ -5748,7 +5750,7 @@ static int q2t_add_target(scsi_qla_host_t *ha)
 
 	sBUG_ON((ha->q2t_tgt != NULL) || (ha->tgt != NULL));
 
-	tgt = kzalloc(L1_CACHE_ALIGN(sizeof(*tgt)), GFP_KERNEL);
+	tgt = kmem_cache_zalloc(q2t_tgt_cachep, GFP_KERNEL);
 	if (tgt == NULL) {
 		PRINT_ERROR("qla2x00t: %s", "Allocation of tgt failed");
 		res = -ENOMEM;
@@ -5864,7 +5866,7 @@ out:
 
 out_free:
 	ha->q2t_tgt = NULL;
-	kfree(tgt);
+	kmem_cache_free(q2t_tgt_cachep, tgt);
 	goto out;
 }
 
@@ -6603,16 +6605,28 @@ static int __init q2t_init(void)
 	PRINT_INFO("qla2x00t: Initializing QLogic Fibre Channel HBA Driver "
 		"target mode addon version %s", Q2T_VERSION_STRING);
 
-	q2t_cmd_cachep = KMEM_CACHE(q2t_cmd, SCST_SLAB_FLAGS);
+	q2t_cmd_cachep = KMEM_CACHE(q2t_cmd, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
 	if (q2t_cmd_cachep == NULL) {
 		res = -ENOMEM;
 		goto out;
 	}
 
+	q2t_sess_cachep = KMEM_CACHE(q2t_sess, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
+	if (q2t_sess_cachep == NULL) {
+		res = -ENOMEM;
+		goto out_cmd_free;
+	}
+
+	q2t_tgt_cachep = KMEM_CACHE(q2t_tgt, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
+	if (q2t_tgt_cachep == NULL) {
+		res = -ENOMEM;
+		goto out_sess_free;
+	}
+
 	q2t_mgmt_cmd_cachep = KMEM_CACHE(q2t_mgmt_cmd, SCST_SLAB_FLAGS);
 	if (q2t_mgmt_cmd_cachep == NULL) {
 		res = -ENOMEM;
-		goto out_cmd_free;
+		goto out_tgt_free;
 	}
 
 	q2t_mgmt_cmd_mempool = mempool_create(25, mempool_alloc_slab,
@@ -6653,6 +6667,12 @@ out_mempool_free:
 out_kmem_free:
 	kmem_cache_destroy(q2t_mgmt_cmd_cachep);
 
+out_tgt_free:
+	kmem_cache_destroy(q2t_tgt_cachep);
+
+out_sess_free:
+	kmem_cache_destroy(q2t_sess_cachep);
+
 out_cmd_free:
 	kmem_cache_destroy(q2t_cmd_cachep);
 	goto out;
@@ -6683,6 +6703,8 @@ static void __exit q2t_exit(void)
 
 	mempool_destroy(q2t_mgmt_cmd_mempool);
 	kmem_cache_destroy(q2t_mgmt_cmd_cachep);
+	kmem_cache_destroy(q2t_tgt_cachep);
+	kmem_cache_destroy(q2t_sess_cachep);
 	kmem_cache_destroy(q2t_cmd_cachep);
 
 	/* Let's make lockdep happy */

scst/src/dev_handlers/scst_user.c

@@ -212,6 +212,8 @@ static int dev_usr_parse(struct scst_cmd *cmd);
 
 /** Data **/
 
+static struct kmem_cache *user_dev_cachep;
+
 static struct kmem_cache *user_cmd_cachep;
 static struct kmem_cache *user_get_cmd_cachep;
 
@@ -1196,8 +1198,8 @@ static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff,
 
 	ucmd->num_data_pages = num_pg;
 
-	ucmd->data_pages = kmalloc(L1_CACHE_ALIGN(sizeof(*ucmd->data_pages) * ucmd->num_data_pages),
-				  GFP_KERNEL);
+	ucmd->data_pages = kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages,
+				   GFP_KERNEL);
 	if (ucmd->data_pages == NULL) {
 		TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array "
 			"(num_data_pages=%d)", ucmd->num_data_pages);
@@ -2930,7 +2932,7 @@ static int dev_user_register_dev(struct file *file,
 		goto out;
 	}
 
-	dev = kzalloc(L1_CACHE_ALIGN(sizeof(*dev)), GFP_KERNEL);
+	dev = kmem_cache_zalloc(user_dev_cachep, GFP_KERNEL);
 	if (dev == NULL) {
 		res = -ENOMEM;
 		goto out_put;
@@ -3079,7 +3081,7 @@ out_free0:
 out_deinit_threads:
 	scst_deinit_threads(&dev->udev_cmd_threads);
 
-	kfree(dev);
+	kmem_cache_free(user_dev_cachep, dev);
 
 out_put:
 	module_put(THIS_MODULE);
@@ -3127,7 +3129,7 @@ static int dev_user_unregister_dev(struct file *file)
 
 	up_write(&dev->dev_rwsem); /* to make lockdep happy */
 
-	kfree(dev);
+	kmem_cache_free(user_dev_cachep, dev);
 
 out_resume:
 	scst_resume_activity();
@@ -3508,7 +3510,7 @@ static int __dev_user_release(void *arg)
 {
 	struct scst_user_dev *dev = arg;
 	dev_user_exit_dev(dev);
-	kfree(dev);
+	kmem_cache_free(user_dev_cachep, dev);
 	return 0;
 }
 
@@ -3788,13 +3790,22 @@ static int __init init_scst_user(void)
 #endif
 #endif
 
-	user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS);
-	if (user_cmd_cachep == NULL) {
+	user_dev_cachep = KMEM_CACHE(scst_user_dev,
+				SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
+	if (user_dev_cachep == NULL) {
 		res = -ENOMEM;
 		goto out;
 	}
 
-	user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS);
+	user_cmd_cachep = KMEM_CACHE(scst_user_cmd,
+				SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
+	if (user_cmd_cachep == NULL) {
+		res = -ENOMEM;
+		goto out_dev_cache;
+	}
+
+	user_get_cmd_cachep = KMEM_CACHE(max_get_reply,
+				SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
 	if (user_get_cmd_cachep == NULL) {
 		res = -ENOMEM;
 		goto out_cache;
@@ -3889,6 +3900,9 @@ out_cache1:
 
 out_cache:
 	kmem_cache_destroy(user_cmd_cachep);
+
+out_dev_cache:
+	kmem_cache_destroy(user_dev_cachep);
 	goto out;
 }
 
@@ -3917,6 +3931,7 @@ static void __exit exit_scst_user(void)
 
 	kmem_cache_destroy(user_get_cmd_cachep);
 	kmem_cache_destroy(user_cmd_cachep);
+	kmem_cache_destroy(user_dev_cachep);
 
 	TRACE_EXIT();
 	return;

scst/src/dev_handlers/scst_vdisk.c

@@ -1630,7 +1630,7 @@ static struct scatterlist *alloc_sg(size_t size, unsigned off, gfp_t gfp_mask,
 
 	sg_cnt = PAGE_ALIGN(size + off) >> PAGE_SHIFT;
 	sg = sg_cnt <= small_sg_size ? small_sg :
-		kmalloc(L1_CACHE_ALIGN(sg_cnt * sizeof(*sg)), gfp_mask);
+		kmalloc(sg_cnt * sizeof(*sg), gfp_mask);
 	if (!sg)
 		goto out;
 
@@ -3387,7 +3387,7 @@ static struct iovec *vdisk_alloc_iv(struct scst_cmd *cmd,
 		p->iv_count = 0;
 		/* It can't be called in atomic context */
 		p->iv = (iv_count <= ARRAY_SIZE(p->small_iv)) ? p->small_iv :
-			kmalloc(L1_CACHE_ALIGN(sizeof(*p->iv) * iv_count), GFP_KERNEL);
+			kmalloc(sizeof(*p->iv) * iv_count, GFP_KERNEL);
 		if (p->iv == NULL) {
 			PRINT_ERROR("Unable to allocate iv (%d)", iv_count);
 			goto out;
@@ -4329,7 +4329,8 @@ static int vdev_create(struct scst_dev_type *devt,
 	if (vdev_find(name))
 		goto out;
 
-	virt_dev = kzalloc(L1_CACHE_ALIGN(sizeof(*virt_dev)), GFP_KERNEL);
+	/* It's read-mostly, so cache alignment isn't needed */
+	virt_dev = kzalloc(sizeof(*virt_dev), GFP_KERNEL);
 	if (virt_dev == NULL) {
 		PRINT_ERROR("Allocation of virtual device %s failed",
 			devt->name);
@@ -6310,13 +6311,15 @@ static int __init init_scst_vdisk_driver(void)
 	init_ops(blockio_ops, ARRAY_SIZE(blockio_ops));
 	init_ops(nullio_ops, ARRAY_SIZE(nullio_ops));
 
-	vdisk_cmd_param_cachep = KMEM_CACHE(vdisk_cmd_params, SCST_SLAB_FLAGS);
+	vdisk_cmd_param_cachep = KMEM_CACHE(vdisk_cmd_params,
+					SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
 	if (vdisk_cmd_param_cachep == NULL) {
 		res = -ENOMEM;
 		goto out;
 	}
 
-	blockio_work_cachep = KMEM_CACHE(scst_blockio_work, SCST_SLAB_FLAGS);
+	blockio_work_cachep = KMEM_CACHE(scst_blockio_work,
+				SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
 	if (blockio_work_cachep == NULL) {
 		res = -ENOMEM;
 		goto out_free_vdisk_cache;

scst/src/scst_lib.c

@@ -3296,7 +3296,7 @@ int scst_alloc_tgt(struct scst_tgt_template *tgtt, struct scst_tgt **tgt)
 
 	TRACE_ENTRY();
 
-	t = kzalloc(sizeof(*t), GFP_KERNEL);
+	t = kmem_cache_zalloc(scst_tgt_cachep, GFP_KERNEL);
 	if (t == NULL) {
 		PRINT_ERROR("%s", "Allocation of tgt failed");
 		res = -ENOMEM;
@@ -3343,7 +3343,7 @@ void scst_free_tgt(struct scst_tgt *tgt)
 	kfree(tgt->default_group_name);
 #endif
 
-	kfree(tgt);
+	kmem_cache_free(scst_tgt_cachep, tgt);
 
 	TRACE_EXIT();
 	return;
@@ -3373,7 +3373,7 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev)
 
 	TRACE_ENTRY();
 
-	dev = kzalloc(L1_CACHE_ALIGN(sizeof(*dev)), gfp_mask);
+	dev = kmem_cache_zalloc(scst_dev_cachep, gfp_mask);
 	if (dev == NULL) {
 		PRINT_ERROR("%s", "Allocation of scst_device failed");
 		res = -ENOMEM;
@@ -3427,7 +3427,7 @@ void scst_free_device(struct scst_device *dev)
 	scst_deinit_threads(&dev->dev_cmd_threads);
 
 	kfree(dev->virt_name);
-	kfree(dev);
+	kmem_cache_free(scst_dev_cachep, dev);
 
 	TRACE_EXIT();
 	return;
@@ -5247,10 +5247,10 @@ void scst_cmd_set_ext_cdb(struct scst_cmd *cmd,
 		goto out;
 	}
 
-	cmd->cdb = kmalloc(L1_CACHE_ALIGN(len), gfp_mask);
+	/* It's read-mostly, so cache alignment isn't needed */
+	cmd->cdb = kmalloc(len, gfp_mask);
 	if (unlikely(cmd->cdb == NULL)) {
-		PRINT_ERROR("Unable to alloc extended CDB (size %d)",
-			L1_CACHE_ALIGN(len));
+		PRINT_ERROR("Unable to alloc extended CDB (size %d)", len);
 		goto out_err;
 	}
 
@@ -5324,10 +5324,11 @@ int scst_pre_init_cmd(struct scst_cmd *cmd, const uint8_t *cdb,
 			res = -EINVAL;
 			goto out;
 		}
-		cmd->cdb = kmalloc(L1_CACHE_ALIGN(cdb_len), gfp_mask);
+		/* It's read-mostly, so cache alignment isn't needed */
+		cmd->cdb = kmalloc(cdb_len, gfp_mask);
 		if (unlikely(cmd->cdb == NULL)) {
 			PRINT_ERROR("Unable to alloc extended CDB (size %d)",
-				L1_CACHE_ALIGN(cdb_len));
+				cdb_len);
 			res = -ENOMEM;
 			goto out;
 		}
@@ -9063,7 +9064,7 @@ int __init scst_lib_init(void)
 	scsi_io_context_cache = kmem_cache_create("scst_scsi_io_context",
 					sizeof(struct scsi_io_context),
 					__alignof__(struct scsi_io_context),
-					0, NULL);
+					SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN, NULL);
 	if (!scsi_io_context_cache) {
 		PRINT_ERROR("%s", "Can't init scsi io context cache");
 		res = -ENOMEM;

scst/src/scst_main.c

@@ -107,6 +107,8 @@ static struct kmem_cache *scst_sense_cachep;
 mempool_t *scst_sense_mempool;
 static struct kmem_cache *scst_aen_cachep;
 mempool_t *scst_aen_mempool;
+struct kmem_cache *scst_tgt_cachep;
+struct kmem_cache *scst_dev_cachep;
 struct kmem_cache *scst_tgtd_cachep;
 struct kmem_cache *scst_sess_cachep;
 struct kmem_cache *scst_acgd_cachep;
@@ -2268,6 +2270,7 @@ static int __init init_scst(void)
 		scst_threads = scst_num_cpus;
 	}
 
+/* Used for rarely used or read-mostly on fast path structures */
 #define INIT_CACHEP(p, s, o) do {					\
 		p = KMEM_CACHE(s, SCST_SLAB_FLAGS);			\
 		TRACE_MEM("Slab create: %s at %p size %zd", #s, p,	\
@@ -2278,6 +2281,17 @@ static int __init init_scst(void)
 		}							\
 	} while (0)
 
+/* Used for structures with fast path write access */
+#define INIT_CACHEP_ALIGN(p, s, o) do {					\
+		p = KMEM_CACHE(s, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);	\
+		TRACE_MEM("Slab create: %s at %p size %zd", #s, p,	\
+			  sizeof(struct s));				\
+		if (p == NULL) {					\
+			res = -ENOMEM;					\
+			goto o;						\
+		}							\
+	} while (0)
+
 	INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out_lib_exit);
 	INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
 			out_destroy_mgmt_cache);
@@ -2289,10 +2303,13 @@ static int __init init_scst(void)
 			    out_destroy_ua_cache);
 	}
 	INIT_CACHEP(scst_aen_cachep, scst_aen, out_destroy_sense_cache);
-	INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache);
-	INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
-	INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
-	INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
+	INIT_CACHEP_ALIGN(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache);
+	INIT_CACHEP_ALIGN(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
+	INIT_CACHEP_ALIGN(scst_dev_cachep, scst_device, out_destroy_sess_cache);
+	INIT_CACHEP_ALIGN(scst_tgt_cachep, scst_tgt, out_destroy_dev_cache);
+	/* They are read-mostly */
+	INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_tgt_cache);
+	INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgtd_cache);
 
 	scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
 		mempool_free_slab, scst_mgmt_cachep);
@@ -2452,9 +2469,15 @@ out_destroy_mgmt_mempool:
 out_destroy_acg_cache:
 	kmem_cache_destroy(scst_acgd_cachep);
 
-out_destroy_tgt_cache:
+out_destroy_tgtd_cache:
 	kmem_cache_destroy(scst_tgtd_cachep);
 
+out_destroy_tgt_cache:
+	kmem_cache_destroy(scst_tgt_cachep);
+
+out_destroy_dev_cache:
+	kmem_cache_destroy(scst_dev_cachep);
+
 out_destroy_sess_cache:
 	kmem_cache_destroy(scst_sess_cachep);
 
@@ -2528,6 +2551,8 @@ static void __exit exit_scst(void)
 	DEINIT_CACHEP(scst_cmd_cachep);
 	DEINIT_CACHEP(scst_sess_cachep);
 	DEINIT_CACHEP(scst_tgtd_cachep);
+	DEINIT_CACHEP(scst_dev_cachep);
+	DEINIT_CACHEP(scst_tgt_cachep);
 	DEINIT_CACHEP(scst_acgd_cachep);
 
 	scst_lib_exit();

scst/src/scst_mem.c

@@ -69,6 +69,8 @@ static struct shrinker *sgv_shrinker;
 static struct shrinker sgv_shrinker;
 #endif
 
+static struct kmem_cache *sgv_pool_cachep;
+
 /*
  * Protected by sgv_pools_mutex AND sgv_pools_lock for writes,
  * either one for reads.
@@ -660,7 +662,7 @@ static int sgv_alloc_arrays(struct sgv_pool_obj *obj,
 
 	sz = pages_to_alloc * sizeof(obj->sg_entries[0]);
 
-	obj->sg_entries = kmalloc(L1_CACHE_ALIGN(sz), gfp_mask);
+	obj->sg_entries = kmalloc(sz, gfp_mask);
 	if (unlikely(obj->sg_entries == NULL)) {
 		TRACE(TRACE_OUT_OF_MEM, "Allocation of sgv_pool_obj "
 			"SG vector failed (size %d)", sz);
@@ -680,7 +682,7 @@ static int sgv_alloc_arrays(struct sgv_pool_obj *obj,
 			 */
 		} else {
 			tsz = pages_to_alloc * sizeof(obj->trans_tbl[0]);
-			obj->trans_tbl = kzalloc(L1_CACHE_ALIGN(tsz), gfp_mask);
+			obj->trans_tbl = kzalloc(tsz, gfp_mask);
 			if (unlikely(obj->trans_tbl == NULL)) {
 				TRACE(TRACE_OUT_OF_MEM, "Allocation of "
 					"trans_tbl failed (size %d)", tsz);
@@ -1021,7 +1023,7 @@ struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size,
 
 		sz = sizeof(*obj) + pages * sizeof(obj->sg_entries[0]);
 
-		obj = kmalloc(L1_CACHE_ALIGN(sz), gfp_mask);
+		obj = kmalloc(sz, gfp_mask);
 		if (unlikely(obj == NULL)) {
 			TRACE(TRACE_OUT_OF_MEM, "Allocation of "
 				"sgv_pool_obj failed (size %d)", size);
@@ -1571,7 +1573,7 @@ static void sgv_pool_destroy(struct sgv_pool *pool)
 		pool->caches[i] = NULL;
 	}
 
-	kfree(pool);
+	kmem_cache_free(sgv_pool_cachep, pool);
 
 	TRACE_EXIT();
 	return;
@@ -1651,7 +1653,7 @@ struct sgv_pool *sgv_pool_create(const char *name,
 		}
 	}
 
-	pool = kzalloc(L1_CACHE_ALIGN(sizeof(*pool)), GFP_KERNEL);
+	pool = kmem_cache_zalloc(sgv_pool_cachep, GFP_KERNEL);
 	if (pool == NULL) {
 		PRINT_ERROR("Allocation of sgv_pool failed (size %zd)",
 			sizeof(*pool));
@@ -1670,7 +1672,7 @@ out_unlock:
 	return pool;
 
 out_free:
-	kfree(pool);
+	kmem_cache_free(sgv_pool_cachep, pool);
 	pool = NULL;
 	goto out_unlock;
 }
@@ -1732,6 +1734,10 @@ int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark)
 
 	TRACE_ENTRY();
 
+	sgv_pool_cachep = KMEM_CACHE(sgv_pool, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN);
+	if (sgv_pool_cachep == NULL)
+		goto out_err;
+
 	sgv_hi_wmk = mem_hwmark;
 	sgv_lo_wmk = mem_lwmark;
 
@@ -1739,7 +1745,7 @@ int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark)
 
 	sgv_norm_pool = sgv_pool_create("sgv", sgv_no_clustering, 0, false, 0);
 	if (sgv_norm_pool == NULL)
-		goto out_err;
+		goto out_free_pool;
 
 	sgv_norm_clust_pool = sgv_pool_create("sgv-clust",
 		sgv_full_clustering, 0, false, 0);
@@ -1769,6 +1775,9 @@ out_free_clust:
 out_free_norm:
 	sgv_pool_destroy(sgv_norm_pool);
 
+out_free_pool:
+	kmem_cache_destroy(sgv_pool_cachep);
+
 out_err:
 	res = -ENOMEM;
 	goto out;
@@ -1790,6 +1799,8 @@ void scst_sgv_pools_deinit(void)
 
 	flush_scheduled_work();
 
+	kmem_cache_destroy(sgv_pool_cachep);
+
 	TRACE_EXIT();
 	return;
 }

scst/src/scst_priv.h

@@ -135,6 +135,8 @@ extern mempool_t *scst_aen_mempool;
 
 extern struct kmem_cache *scst_cmd_cachep;
 extern struct kmem_cache *scst_sess_cachep;
+extern struct kmem_cache *scst_dev_cachep;
+extern struct kmem_cache *scst_tgt_cachep;
 extern struct kmem_cache *scst_tgtd_cachep;
 extern struct kmem_cache *scst_acgd_cachep;
 

scst_local/scst_local.c

@@ -1747,7 +1747,8 @@ static int __scst_local_add_adapter(struct scst_local_tgt *tgt,
 
 	TRACE_ENTRY();
 
-	sess = kzalloc(L1_CACHE_ALIGN(sizeof(*sess)), GFP_KERNEL);
+	/* It's read-mostly, so cache alignment isn't needed */
+	sess = kzalloc(sizeof(*sess), GFP_KERNEL);
 	if (NULL == sess) {
 		PRINT_ERROR("Unable to alloc scst_lcl_host (size %zu)",
 			sizeof(*sess));