diff --git a/iscsi-scst/kernel/conn.c b/iscsi-scst/kernel/conn.c index 5f3bc93a0..b06b0ae5b 100644 --- a/iscsi-scst/kernel/conn.c +++ b/iscsi-scst/kernel/conn.c @@ -833,7 +833,7 @@ int conn_free(struct iscsi_conn *conn) free_page((unsigned long)conn->read_iov); - kfree(conn); + kmem_cache_free(iscsi_conn_cache, conn); if (list_empty(&session->conn_list)) { sBUG_ON(session->sess_reinst_successor != NULL); @@ -850,7 +850,7 @@ static int iscsi_conn_alloc(struct iscsi_session *session, struct iscsi_conn *conn; int res = 0; - conn = kzalloc(L1_CACHE_ALIGN(sizeof(*conn)), GFP_KERNEL); + conn = kmem_cache_zalloc(iscsi_conn_cache, GFP_KERNEL); if (!conn) { res = -ENOMEM; goto out_err; @@ -941,7 +941,7 @@ out_free_iov: free_page((unsigned long)conn->read_iov); out_err_free_conn: - kfree(conn); + kmem_cache_free(iscsi_conn_cache, conn); out_err: goto out; diff --git a/iscsi-scst/kernel/iscsi.c b/iscsi-scst/kernel/iscsi.c index a03e3734c..cad0c0b9f 100644 --- a/iscsi-scst/kernel/iscsi.c +++ b/iscsi-scst/kernel/iscsi.c @@ -51,8 +51,13 @@ static struct kmem_cache *iscsi_cmnd_cache; static DEFINE_MUTEX(iscsi_threads_pool_mutex); static LIST_HEAD(iscsi_thread_pools_list); +static struct kmem_cache *iscsi_thread_pool_cache; + static struct iscsi_thread_pool *iscsi_main_thread_pool; +struct kmem_cache *iscsi_conn_cache; +struct kmem_cache *iscsi_sess_cache; + static struct page *dummy_page; static struct scatterlist dummy_sg; @@ -3892,6 +3897,48 @@ struct scst_tgt_template iscsi_template = { .get_scsi_transport_version = iscsi_get_scsi_transport_version, }; +static void __iscsi_threads_pool_put(struct iscsi_thread_pool *p) +{ + struct iscsi_thread *t, *tt; + + TRACE_ENTRY(); + + p->thread_pool_ref--; + if (p->thread_pool_ref > 0) { + TRACE_DBG("iSCSI thread pool %p still has %d references)", + p, p->thread_pool_ref); + goto out; + } + + TRACE_DBG("Freeing iSCSI thread pool %p", p); + + list_for_each_entry_safe(t, tt, &p->threads_list, threads_list_entry) { + kthread_stop(t->thr); + list_del(&t->threads_list_entry); + kfree(t); + } + + list_del(&p->thread_pools_list_entry); + + kmem_cache_free(iscsi_thread_pool_cache, p); + +out: + TRACE_EXIT(); + return; +} + +void iscsi_threads_pool_put(struct iscsi_thread_pool *p) +{ + TRACE_ENTRY(); + + mutex_lock(&iscsi_threads_pool_mutex); + __iscsi_threads_pool_put(p); + mutex_unlock(&iscsi_threads_pool_mutex); + + TRACE_EXIT(); + return; +} + int iscsi_threads_pool_get(const cpumask_t *cpu_mask, struct iscsi_thread_pool **out_pool) { @@ -3918,7 +3965,7 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask, TRACE_DBG("%s", "Creating new iSCSI thread pool"); - p = kzalloc(L1_CACHE_ALIGN(sizeof(*p)), GFP_KERNEL); + p = kmem_cache_zalloc(iscsi_thread_pool_cache, GFP_KERNEL); if (p == NULL) { PRINT_ERROR("Unable to allocate iSCSI thread pool (size %zd)", sizeof(*p)); @@ -3958,6 +4005,8 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask, count++; } + list_add_tail(&p->thread_pools_list_entry, &iscsi_thread_pools_list); + for (j = 0; j < 2; j++) { int (*fn)(void *); char name[25]; @@ -4004,7 +4053,6 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask, } } - list_add_tail(&p->thread_pools_list_entry, &iscsi_thread_pools_list); res = 0; TRACE_DBG("Created iSCSI thread pool %p", p); @@ -4012,55 +4060,17 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask, out_unlock: mutex_unlock(&iscsi_threads_pool_mutex); - if (out_pool != NULL) - *out_pool = p; + *out_pool = p; TRACE_EXIT_RES(res); return res; out_free: - list_for_each_entry_safe(t, tt, &p->threads_list, threads_list_entry) { - kthread_stop(t->thr); - list_del(&t->threads_list_entry); - kfree(t); - } + __iscsi_threads_pool_put(p); + p = NULL; goto out_unlock; } -void iscsi_threads_pool_put(struct iscsi_thread_pool *p) -{ - struct iscsi_thread *t, *tt; - - TRACE_ENTRY(); - - mutex_lock(&iscsi_threads_pool_mutex); - - p->thread_pool_ref--; - if (p->thread_pool_ref > 0) { - TRACE_DBG("iSCSI thread pool %p still has %d references)", - p, p->thread_pool_ref); - goto out_unlock; - } - - TRACE_DBG("Freeing iSCSI thread pool %p", p); - - list_for_each_entry_safe(t, tt, &p->threads_list, threads_list_entry) { - kthread_stop(t->thr); - list_del(&t->threads_list_entry); - kfree(t); - } - - list_del(&p->thread_pools_list_entry); - - kfree(p); - -out_unlock: - mutex_unlock(&iscsi_threads_pool_mutex); - - TRACE_EXIT(); - return; -} - static int __init iscsi_init(void) { int err = 0; @@ -4111,12 +4121,32 @@ static int __init iscsi_init(void) if (err < 0) goto out_reg; - iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS); + iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); if (!iscsi_cmnd_cache) { err = -ENOMEM; goto out_event; } + iscsi_thread_pool_cache = KMEM_CACHE(iscsi_thread_pool, + SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); + if (!iscsi_thread_pool_cache) { + err = -ENOMEM; + goto out_kmem_cmd; + } + + iscsi_conn_cache = KMEM_CACHE(iscsi_conn, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); + if (!iscsi_conn_cache) { + err = -ENOMEM; + goto out_kmem_tp; + } + + iscsi_sess_cache = KMEM_CACHE(iscsi_session, + SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); + if (!iscsi_sess_cache) { + err = -ENOMEM; + goto out_kmem_conn; + } + err = scst_register_target_template(&iscsi_template); if (err < 0) goto out_kmem; @@ -4147,6 +4177,15 @@ out_reg_tmpl: scst_unregister_target_template(&iscsi_template); out_kmem: + kmem_cache_destroy(iscsi_sess_cache); + +out_kmem_conn: + kmem_cache_destroy(iscsi_conn_cache); + +out_kmem_tp: + kmem_cache_destroy(iscsi_thread_pool_cache); + +out_kmem_cmd: kmem_cache_destroy(iscsi_cmnd_cache); out_event: @@ -4181,6 +4220,9 @@ static void __exit iscsi_exit(void) #endif event_exit(); + kmem_cache_destroy(iscsi_sess_cache); + kmem_cache_destroy(iscsi_conn_cache); + kmem_cache_destroy(iscsi_thread_pool_cache); kmem_cache_destroy(iscsi_cmnd_cache); scst_unregister_target_template(&iscsi_template); diff --git a/iscsi-scst/kernel/iscsi.h b/iscsi-scst/kernel/iscsi.h index e14f1fd5f..7b7b05f07 100644 --- a/iscsi-scst/kernel/iscsi.h +++ b/iscsi-scst/kernel/iscsi.h @@ -518,6 +518,9 @@ extern struct mutex target_mgmt_mutex; extern int ctr_open_state; extern const struct file_operations ctr_fops; +extern struct kmem_cache *iscsi_conn_cache; +extern struct kmem_cache *iscsi_sess_cache; + /* iscsi.c */ extern struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *, struct iscsi_cmnd *parent); diff --git a/iscsi-scst/kernel/session.c b/iscsi-scst/kernel/session.c index 7a6ba0175..a552b8efe 100644 --- a/iscsi-scst/kernel/session.c +++ b/iscsi-scst/kernel/session.c @@ -43,7 +43,7 @@ static int iscsi_session_alloc(struct iscsi_target *target, struct iscsi_session *session; char *name = NULL; - session = kzalloc(L1_CACHE_ALIGN(sizeof(*session)), GFP_KERNEL); + session = kmem_cache_zalloc(iscsi_sess_cache, GFP_KERNEL); if (!session) return -ENOMEM; @@ -114,7 +114,7 @@ err_unreg: err: if (session) { kfree(session->initiator_name); - kfree(session); + kmem_cache_free(iscsi_sess_cache, session); #ifdef CONFIG_SCST_PROC kfree(name); #endif @@ -277,7 +277,7 @@ out_err_unlock: static void __session_free(struct iscsi_session *session) { kfree(session->initiator_name); - kfree(session); + kmem_cache_free(iscsi_sess_cache, session); } static void iscsi_unreg_sess_done(struct scst_session *scst_sess) diff --git a/qla2x00t/qla2x00-target/qla2x00t.c b/qla2x00t/qla2x00-target/qla2x00t.c index ae7b65631..c03220558 100644 --- a/qla2x00t/qla2x00-target/qla2x00t.c +++ b/qla2x00t/qla2x00-target/qla2x00t.c @@ -242,6 +242,8 @@ static struct scst_tgt_template tgt2x_template = { }; static struct kmem_cache *q2t_cmd_cachep; +static struct kmem_cache *q2t_sess_cachep; +static struct kmem_cache *q2t_tgt_cachep; static struct kmem_cache *q2t_mgmt_cmd_cachep; static mempool_t *q2t_mgmt_cmd_mempool; @@ -629,7 +631,7 @@ static void q2t_free_session_done(struct scst_session *scst_sess) TRACE_MGMT_DBG("Unregistration of sess %p finished", sess); - kfree(sess); + kmem_cache_free(q2t_sess_cachep, sess); if (tgt == NULL) goto out; @@ -1155,7 +1157,7 @@ static struct q2t_sess *q2t_create_sess(scsi_qla_host_t *ha, fc_port_t *fcport, /* We are under tgt_mutex, so a new sess can't be added behind us */ - sess = kzalloc(L1_CACHE_ALIGN(sizeof(*sess)), GFP_KERNEL); + sess = kmem_cache_zalloc(q2t_sess_cachep, GFP_KERNEL); if (sess == NULL) { PRINT_ERROR("qla2x00t(%ld): session allocation failed, " "all commands from port %02x:%02x:%02x:%02x:" @@ -1229,7 +1231,7 @@ out_free_sess_wwn: /* go through */ out_free_sess: - kfree(sess); + kmem_cache_free(q2t_sess_cachep, sess); sess = NULL; goto out; } @@ -1442,7 +1444,7 @@ static int q2t_target_release(struct scst_tgt *scst_tgt) TRACE_MGMT_DBG("Release of tgt %p finished", tgt); - kfree(tgt); + kmem_cache_free(q2t_tgt_cachep, tgt); TRACE_EXIT(); return 0; @@ -5748,7 +5750,7 @@ static int q2t_add_target(scsi_qla_host_t *ha) sBUG_ON((ha->q2t_tgt != NULL) || (ha->tgt != NULL)); - tgt = kzalloc(L1_CACHE_ALIGN(sizeof(*tgt)), GFP_KERNEL); + tgt = kmem_cache_zalloc(q2t_tgt_cachep, GFP_KERNEL); if (tgt == NULL) { PRINT_ERROR("qla2x00t: %s", "Allocation of tgt failed"); res = -ENOMEM; @@ -5864,7 +5866,7 @@ out: out_free: ha->q2t_tgt = NULL; - kfree(tgt); + kmem_cache_free(q2t_tgt_cachep, tgt); goto out; } @@ -6603,16 +6605,28 @@ static int __init q2t_init(void) PRINT_INFO("qla2x00t: Initializing QLogic Fibre Channel HBA Driver " "target mode addon version %s", Q2T_VERSION_STRING); - q2t_cmd_cachep = KMEM_CACHE(q2t_cmd, SCST_SLAB_FLAGS); + q2t_cmd_cachep = KMEM_CACHE(q2t_cmd, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); if (q2t_cmd_cachep == NULL) { res = -ENOMEM; goto out; } + q2t_sess_cachep = KMEM_CACHE(q2t_sess, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); + if (q2t_sess_cachep == NULL) { + res = -ENOMEM; + goto out_cmd_free; + } + + q2t_tgt_cachep = KMEM_CACHE(q2t_tgt, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); + if (q2t_tgt_cachep == NULL) { + res = -ENOMEM; + goto out_sess_free; + } + q2t_mgmt_cmd_cachep = KMEM_CACHE(q2t_mgmt_cmd, SCST_SLAB_FLAGS); if (q2t_mgmt_cmd_cachep == NULL) { res = -ENOMEM; - goto out_cmd_free; + goto out_tgt_free; } q2t_mgmt_cmd_mempool = mempool_create(25, mempool_alloc_slab, @@ -6653,6 +6667,12 @@ out_mempool_free: out_kmem_free: kmem_cache_destroy(q2t_mgmt_cmd_cachep); +out_tgt_free: + kmem_cache_destroy(q2t_tgt_cachep); + +out_sess_free: + kmem_cache_destroy(q2t_sess_cachep); + out_cmd_free: kmem_cache_destroy(q2t_cmd_cachep); goto out; @@ -6683,6 +6703,8 @@ static void __exit q2t_exit(void) mempool_destroy(q2t_mgmt_cmd_mempool); kmem_cache_destroy(q2t_mgmt_cmd_cachep); + kmem_cache_destroy(q2t_tgt_cachep); + kmem_cache_destroy(q2t_sess_cachep); kmem_cache_destroy(q2t_cmd_cachep); /* Let's make lockdep happy */ diff --git a/scst/src/dev_handlers/scst_user.c b/scst/src/dev_handlers/scst_user.c index e3477ec98..49577a536 100644 --- a/scst/src/dev_handlers/scst_user.c +++ b/scst/src/dev_handlers/scst_user.c @@ -212,6 +212,8 @@ static int dev_usr_parse(struct scst_cmd *cmd); /** Data **/ +static struct kmem_cache *user_dev_cachep; + static struct kmem_cache *user_cmd_cachep; static struct kmem_cache *user_get_cmd_cachep; @@ -1196,8 +1198,8 @@ static int dev_user_map_buf(struct scst_user_cmd *ucmd, unsigned long ubuff, ucmd->num_data_pages = num_pg; - ucmd->data_pages = kmalloc(L1_CACHE_ALIGN(sizeof(*ucmd->data_pages) * ucmd->num_data_pages), - GFP_KERNEL); + ucmd->data_pages = kmalloc(sizeof(*ucmd->data_pages) * ucmd->num_data_pages, + GFP_KERNEL); if (ucmd->data_pages == NULL) { TRACE(TRACE_OUT_OF_MEM, "Unable to allocate data_pages array " "(num_data_pages=%d)", ucmd->num_data_pages); @@ -2930,7 +2932,7 @@ static int dev_user_register_dev(struct file *file, goto out; } - dev = kzalloc(L1_CACHE_ALIGN(sizeof(*dev)), GFP_KERNEL); + dev = kmem_cache_zalloc(user_dev_cachep, GFP_KERNEL); if (dev == NULL) { res = -ENOMEM; goto out_put; @@ -3079,7 +3081,7 @@ out_free0: out_deinit_threads: scst_deinit_threads(&dev->udev_cmd_threads); - kfree(dev); + kmem_cache_free(user_dev_cachep, dev); out_put: module_put(THIS_MODULE); @@ -3127,7 +3129,7 @@ static int dev_user_unregister_dev(struct file *file) up_write(&dev->dev_rwsem); /* to make lockdep happy */ - kfree(dev); + kmem_cache_free(user_dev_cachep, dev); out_resume: scst_resume_activity(); @@ -3508,7 +3510,7 @@ static int __dev_user_release(void *arg) { struct scst_user_dev *dev = arg; dev_user_exit_dev(dev); - kfree(dev); + kmem_cache_free(user_dev_cachep, dev); return 0; } @@ -3788,13 +3790,22 @@ static int __init init_scst_user(void) #endif #endif - user_cmd_cachep = KMEM_CACHE(scst_user_cmd, SCST_SLAB_FLAGS); - if (user_cmd_cachep == NULL) { + user_dev_cachep = KMEM_CACHE(scst_user_dev, + SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); + if (user_dev_cachep == NULL) { res = -ENOMEM; goto out; } - user_get_cmd_cachep = KMEM_CACHE(max_get_reply, SCST_SLAB_FLAGS); + user_cmd_cachep = KMEM_CACHE(scst_user_cmd, + SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); + if (user_cmd_cachep == NULL) { + res = -ENOMEM; + goto out_dev_cache; + } + + user_get_cmd_cachep = KMEM_CACHE(max_get_reply, + SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); if (user_get_cmd_cachep == NULL) { res = -ENOMEM; goto out_cache; @@ -3889,6 +3900,9 @@ out_cache1: out_cache: kmem_cache_destroy(user_cmd_cachep); + +out_dev_cache: + kmem_cache_destroy(user_dev_cachep); goto out; } @@ -3917,6 +3931,7 @@ static void __exit exit_scst_user(void) kmem_cache_destroy(user_get_cmd_cachep); kmem_cache_destroy(user_cmd_cachep); + kmem_cache_destroy(user_dev_cachep); TRACE_EXIT(); return; diff --git a/scst/src/dev_handlers/scst_vdisk.c b/scst/src/dev_handlers/scst_vdisk.c index 269fd80a9..b4e4ae534 100644 --- a/scst/src/dev_handlers/scst_vdisk.c +++ b/scst/src/dev_handlers/scst_vdisk.c @@ -1630,7 +1630,7 @@ static struct scatterlist *alloc_sg(size_t size, unsigned off, gfp_t gfp_mask, sg_cnt = PAGE_ALIGN(size + off) >> PAGE_SHIFT; sg = sg_cnt <= small_sg_size ? small_sg : - kmalloc(L1_CACHE_ALIGN(sg_cnt * sizeof(*sg)), gfp_mask); + kmalloc(sg_cnt * sizeof(*sg), gfp_mask); if (!sg) goto out; @@ -3387,7 +3387,7 @@ static struct iovec *vdisk_alloc_iv(struct scst_cmd *cmd, p->iv_count = 0; /* It can't be called in atomic context */ p->iv = (iv_count <= ARRAY_SIZE(p->small_iv)) ? p->small_iv : - kmalloc(L1_CACHE_ALIGN(sizeof(*p->iv) * iv_count), GFP_KERNEL); + kmalloc(sizeof(*p->iv) * iv_count, GFP_KERNEL); if (p->iv == NULL) { PRINT_ERROR("Unable to allocate iv (%d)", iv_count); goto out; @@ -4329,7 +4329,8 @@ static int vdev_create(struct scst_dev_type *devt, if (vdev_find(name)) goto out; - virt_dev = kzalloc(L1_CACHE_ALIGN(sizeof(*virt_dev)), GFP_KERNEL); + /* It's read-mostly, so cache alignment isn't needed */ + virt_dev = kzalloc(sizeof(*virt_dev), GFP_KERNEL); if (virt_dev == NULL) { PRINT_ERROR("Allocation of virtual device %s failed", devt->name); @@ -6310,13 +6311,15 @@ static int __init init_scst_vdisk_driver(void) init_ops(blockio_ops, ARRAY_SIZE(blockio_ops)); init_ops(nullio_ops, ARRAY_SIZE(nullio_ops)); - vdisk_cmd_param_cachep = KMEM_CACHE(vdisk_cmd_params, SCST_SLAB_FLAGS); + vdisk_cmd_param_cachep = KMEM_CACHE(vdisk_cmd_params, + SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); if (vdisk_cmd_param_cachep == NULL) { res = -ENOMEM; goto out; } - blockio_work_cachep = KMEM_CACHE(scst_blockio_work, SCST_SLAB_FLAGS); + blockio_work_cachep = KMEM_CACHE(scst_blockio_work, + SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); if (blockio_work_cachep == NULL) { res = -ENOMEM; goto out_free_vdisk_cache; diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index 1f6408fa4..cdce21d7a 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -3296,7 +3296,7 @@ int scst_alloc_tgt(struct scst_tgt_template *tgtt, struct scst_tgt **tgt) TRACE_ENTRY(); - t = kzalloc(sizeof(*t), GFP_KERNEL); + t = kmem_cache_zalloc(scst_tgt_cachep, GFP_KERNEL); if (t == NULL) { PRINT_ERROR("%s", "Allocation of tgt failed"); res = -ENOMEM; @@ -3343,7 +3343,7 @@ void scst_free_tgt(struct scst_tgt *tgt) kfree(tgt->default_group_name); #endif - kfree(tgt); + kmem_cache_free(scst_tgt_cachep, tgt); TRACE_EXIT(); return; @@ -3373,7 +3373,7 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev) TRACE_ENTRY(); - dev = kzalloc(L1_CACHE_ALIGN(sizeof(*dev)), gfp_mask); + dev = kmem_cache_zalloc(scst_dev_cachep, gfp_mask); if (dev == NULL) { PRINT_ERROR("%s", "Allocation of scst_device failed"); res = -ENOMEM; @@ -3427,7 +3427,7 @@ void scst_free_device(struct scst_device *dev) scst_deinit_threads(&dev->dev_cmd_threads); kfree(dev->virt_name); - kfree(dev); + kmem_cache_free(scst_dev_cachep, dev); TRACE_EXIT(); return; @@ -5247,10 +5247,10 @@ void scst_cmd_set_ext_cdb(struct scst_cmd *cmd, goto out; } - cmd->cdb = kmalloc(L1_CACHE_ALIGN(len), gfp_mask); + /* It's read-mostly, so cache alignment isn't needed */ + cmd->cdb = kmalloc(len, gfp_mask); if (unlikely(cmd->cdb == NULL)) { - PRINT_ERROR("Unable to alloc extended CDB (size %d)", - L1_CACHE_ALIGN(len)); + PRINT_ERROR("Unable to alloc extended CDB (size %d)", len); goto out_err; } @@ -5324,10 +5324,11 @@ int scst_pre_init_cmd(struct scst_cmd *cmd, const uint8_t *cdb, res = -EINVAL; goto out; } - cmd->cdb = kmalloc(L1_CACHE_ALIGN(cdb_len), gfp_mask); + /* It's read-mostly, so cache alignment isn't needed */ + cmd->cdb = kmalloc(cdb_len, gfp_mask); if (unlikely(cmd->cdb == NULL)) { PRINT_ERROR("Unable to alloc extended CDB (size %d)", - L1_CACHE_ALIGN(cdb_len)); + cdb_len); res = -ENOMEM; goto out; } @@ -9063,7 +9064,7 @@ int __init scst_lib_init(void) scsi_io_context_cache = kmem_cache_create("scst_scsi_io_context", sizeof(struct scsi_io_context), __alignof__(struct scsi_io_context), - 0, NULL); + SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN, NULL); if (!scsi_io_context_cache) { PRINT_ERROR("%s", "Can't init scsi io context cache"); res = -ENOMEM; diff --git a/scst/src/scst_main.c b/scst/src/scst_main.c index 58503dfa3..8fdf27781 100644 --- a/scst/src/scst_main.c +++ b/scst/src/scst_main.c @@ -107,6 +107,8 @@ static struct kmem_cache *scst_sense_cachep; mempool_t *scst_sense_mempool; static struct kmem_cache *scst_aen_cachep; mempool_t *scst_aen_mempool; +struct kmem_cache *scst_tgt_cachep; +struct kmem_cache *scst_dev_cachep; struct kmem_cache *scst_tgtd_cachep; struct kmem_cache *scst_sess_cachep; struct kmem_cache *scst_acgd_cachep; @@ -2268,6 +2270,7 @@ static int __init init_scst(void) scst_threads = scst_num_cpus; } +/* Used for rarely used or read-mostly on fast path structures */ #define INIT_CACHEP(p, s, o) do { \ p = KMEM_CACHE(s, SCST_SLAB_FLAGS); \ TRACE_MEM("Slab create: %s at %p size %zd", #s, p, \ @@ -2278,6 +2281,17 @@ static int __init init_scst(void) } \ } while (0) +/* Used for structures with fast path write access */ +#define INIT_CACHEP_ALIGN(p, s, o) do { \ + p = KMEM_CACHE(s, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); \ + TRACE_MEM("Slab create: %s at %p size %zd", #s, p, \ + sizeof(struct s)); \ + if (p == NULL) { \ + res = -ENOMEM; \ + goto o; \ + } \ + } while (0) + INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out_lib_exit); INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub, out_destroy_mgmt_cache); @@ -2289,10 +2303,13 @@ static int __init init_scst(void) out_destroy_ua_cache); } INIT_CACHEP(scst_aen_cachep, scst_aen, out_destroy_sense_cache); - INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache); - INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache); - INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache); - INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache); + INIT_CACHEP_ALIGN(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache); + INIT_CACHEP_ALIGN(scst_sess_cachep, scst_session, out_destroy_cmd_cache); + INIT_CACHEP_ALIGN(scst_dev_cachep, scst_device, out_destroy_sess_cache); + INIT_CACHEP_ALIGN(scst_tgt_cachep, scst_tgt, out_destroy_dev_cache); + /* They are read-mostly */ + INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_tgt_cache); + INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgtd_cache); scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab, mempool_free_slab, scst_mgmt_cachep); @@ -2452,9 +2469,15 @@ out_destroy_mgmt_mempool: out_destroy_acg_cache: kmem_cache_destroy(scst_acgd_cachep); -out_destroy_tgt_cache: +out_destroy_tgtd_cache: kmem_cache_destroy(scst_tgtd_cachep); +out_destroy_tgt_cache: + kmem_cache_destroy(scst_tgt_cachep); + +out_destroy_dev_cache: + kmem_cache_destroy(scst_dev_cachep); + out_destroy_sess_cache: kmem_cache_destroy(scst_sess_cachep); @@ -2528,6 +2551,8 @@ static void __exit exit_scst(void) DEINIT_CACHEP(scst_cmd_cachep); DEINIT_CACHEP(scst_sess_cachep); DEINIT_CACHEP(scst_tgtd_cachep); + DEINIT_CACHEP(scst_dev_cachep); + DEINIT_CACHEP(scst_tgt_cachep); DEINIT_CACHEP(scst_acgd_cachep); scst_lib_exit(); diff --git a/scst/src/scst_mem.c b/scst/src/scst_mem.c index ca36975ea..b29e9af7a 100644 --- a/scst/src/scst_mem.c +++ b/scst/src/scst_mem.c @@ -69,6 +69,8 @@ static struct shrinker *sgv_shrinker; static struct shrinker sgv_shrinker; #endif +static struct kmem_cache *sgv_pool_cachep; + /* * Protected by sgv_pools_mutex AND sgv_pools_lock for writes, * either one for reads. @@ -660,7 +662,7 @@ static int sgv_alloc_arrays(struct sgv_pool_obj *obj, sz = pages_to_alloc * sizeof(obj->sg_entries[0]); - obj->sg_entries = kmalloc(L1_CACHE_ALIGN(sz), gfp_mask); + obj->sg_entries = kmalloc(sz, gfp_mask); if (unlikely(obj->sg_entries == NULL)) { TRACE(TRACE_OUT_OF_MEM, "Allocation of sgv_pool_obj " "SG vector failed (size %d)", sz); @@ -680,7 +682,7 @@ static int sgv_alloc_arrays(struct sgv_pool_obj *obj, */ } else { tsz = pages_to_alloc * sizeof(obj->trans_tbl[0]); - obj->trans_tbl = kzalloc(L1_CACHE_ALIGN(tsz), gfp_mask); + obj->trans_tbl = kzalloc(tsz, gfp_mask); if (unlikely(obj->trans_tbl == NULL)) { TRACE(TRACE_OUT_OF_MEM, "Allocation of " "trans_tbl failed (size %d)", tsz); @@ -1021,7 +1023,7 @@ struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size, sz = sizeof(*obj) + pages * sizeof(obj->sg_entries[0]); - obj = kmalloc(L1_CACHE_ALIGN(sz), gfp_mask); + obj = kmalloc(sz, gfp_mask); if (unlikely(obj == NULL)) { TRACE(TRACE_OUT_OF_MEM, "Allocation of " "sgv_pool_obj failed (size %d)", size); @@ -1571,7 +1573,7 @@ static void sgv_pool_destroy(struct sgv_pool *pool) pool->caches[i] = NULL; } - kfree(pool); + kmem_cache_free(sgv_pool_cachep, pool); TRACE_EXIT(); return; @@ -1651,7 +1653,7 @@ struct sgv_pool *sgv_pool_create(const char *name, } } - pool = kzalloc(L1_CACHE_ALIGN(sizeof(*pool)), GFP_KERNEL); + pool = kmem_cache_zalloc(sgv_pool_cachep, GFP_KERNEL); if (pool == NULL) { PRINT_ERROR("Allocation of sgv_pool failed (size %zd)", sizeof(*pool)); @@ -1670,7 +1672,7 @@ out_unlock: return pool; out_free: - kfree(pool); + kmem_cache_free(sgv_pool_cachep, pool); pool = NULL; goto out_unlock; } @@ -1732,6 +1734,10 @@ int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark) TRACE_ENTRY(); + sgv_pool_cachep = KMEM_CACHE(sgv_pool, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN); + if (sgv_pool_cachep == NULL) + goto out_err; + sgv_hi_wmk = mem_hwmark; sgv_lo_wmk = mem_lwmark; @@ -1739,7 +1745,7 @@ int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark) sgv_norm_pool = sgv_pool_create("sgv", sgv_no_clustering, 0, false, 0); if (sgv_norm_pool == NULL) - goto out_err; + goto out_free_pool; sgv_norm_clust_pool = sgv_pool_create("sgv-clust", sgv_full_clustering, 0, false, 0); @@ -1769,6 +1775,9 @@ out_free_clust: out_free_norm: sgv_pool_destroy(sgv_norm_pool); +out_free_pool: + kmem_cache_destroy(sgv_pool_cachep); + out_err: res = -ENOMEM; goto out; @@ -1790,6 +1799,8 @@ void scst_sgv_pools_deinit(void) flush_scheduled_work(); + kmem_cache_destroy(sgv_pool_cachep); + TRACE_EXIT(); return; } diff --git a/scst/src/scst_priv.h b/scst/src/scst_priv.h index 27dea3a46..99042cd64 100644 --- a/scst/src/scst_priv.h +++ b/scst/src/scst_priv.h @@ -135,6 +135,8 @@ extern mempool_t *scst_aen_mempool; extern struct kmem_cache *scst_cmd_cachep; extern struct kmem_cache *scst_sess_cachep; +extern struct kmem_cache *scst_dev_cachep; +extern struct kmem_cache *scst_tgt_cachep; extern struct kmem_cache *scst_tgtd_cachep; extern struct kmem_cache *scst_acgd_cachep; diff --git a/scst_local/scst_local.c b/scst_local/scst_local.c index fc12a9541..55e86cca9 100644 --- a/scst_local/scst_local.c +++ b/scst_local/scst_local.c @@ -1747,7 +1747,8 @@ static int __scst_local_add_adapter(struct scst_local_tgt *tgt, TRACE_ENTRY(); - sess = kzalloc(L1_CACHE_ALIGN(sizeof(*sess)), GFP_KERNEL); + /* It's read-mostly, so cache alignment isn't needed */ + sess = kzalloc(sizeof(*sess), GFP_KERNEL); if (NULL == sess) { PRINT_ERROR("Unable to alloc scst_lcl_host (size %zu)", sizeof(*sess));