mirrors/scst
1
0
Fork 0
mirror of https://github.com/SCST-project/scst.git synced 2026-07-19 06:22:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

scst: performance improvements

Browse Source 
This patch trying to allocate memory in per-NUMA and per-CPU manner to
(significantly) improve performance for high IOPS cases.



git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@7075 d57e44dd-8a1f-0410-8b47-8ef2f437770f
This commit is contained in:
Vladislav Bolkhovitin
2016-12-20 06:21:13 +00:00
parent 807836950b
commit bdb23b8923
11 changed files with 400 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scst/README
+7
View File
@@ -501,6 +501,9 @@ following entries:
or -errno for error). The following two shell functions show how to do
this:
- force_global_sgv_pool - if not set, buffers for SCSI commands are
allocated from per-CPU SGV pool. Otherwise, global SGV pool is used.
# Read the SCST sysfs attribute $1. See also scst/README for more information.
scst_sysfs_read() {
local EAGAIN val
@@ -566,6 +569,10 @@ SCST dev handlers can have the following common entries:
- max_tgt_dev_commands - maximum number of SCSI commands any session to
this device can have in flight.
- numa_node_id - NUMA node id this device physically belongs to. SCST
NUMA handling assumes that being used in the system NUMA memory
allocation policy is to always allocate from the current node.
Attribute "block" allows to temporary block and unblock this device.
"Blocking" means that no new commands for this device will go into the
execution stage, but instead will be suspended just before it. The
scst/README_in-tree
+7
View File
@@ -365,6 +365,9 @@ following entries:
or -errno for error). The following two shell functions show how to do
this:
- force_global_sgv_pool - if not set, buffers for SCSI commands are
allocated from per-CPU SGV pool. Otherwise, global SGV pool is used.
# Read the SCST sysfs attribute $1. See also scst/README for more information.
scst_sysfs_read() {
local EAGAIN val
@@ -430,6 +433,10 @@ SCST dev handlers can have the following common entries:
- max_tgt_dev_commands - maximum number of SCSI commands any session to
this device can have in flight.
- numa_node_id - NUMA node id this device physically belongs to. SCST
NUMA handling assumes that being used in the system NUMA memory
allocation policy is to always allocate from the current node.
Attribute "block" allows to temporary block and unblock this device.
"Blocking" means that no new commands for this device will go into the
execution stage, but instead will be suspended just before it. The
scst/include/scst.h
+16 -3
View File
@@ -3039,6 +3039,9 @@ struct scst_device {
/* End of persistent reservation fields protected by dev_pr_mutex. */
/* NUMA node id of this device, if any (default - NUMA_NO_NODE) */
int dev_numa_node_id;
/*
* Count of connected tgt_devs from transports, which don't support
* PRs, i.e. don't have get_initiator_port_transport_id(). Protected
@@ -3138,7 +3141,11 @@ struct scst_tgt_dev {
gfp_t tgt_dev_gfp_mask;
/* SGV pool from which buffers of this tgt_dev's cmds should be allocated */
struct sgv_pool *pool;
#ifdef CONFIG_CPUMASK_OFFSTACK
struct sgv_pool **pools;
#else
struct sgv_pool *pools[NR_CPUS];
#endif
/* Max number of allowed in this tgt_dev SG segments */
int max_sg_cnt;
@@ -3741,8 +3748,14 @@ struct scst_cmd *scst_find_cmd(struct scst_session *sess, void *data,
enum dma_data_direction scst_to_dma_dir(int scst_dir);
enum dma_data_direction scst_to_tgt_dma_dir(int scst_dir);
int scst_register_virtual_device(struct scst_dev_type *dev_handler,
const char *dev_name);
int scst_register_virtual_device_node(struct scst_dev_type *dev_handler,
const char *dev_name, int nodeid);
static inline int scst_register_virtual_device(struct scst_dev_type *dev_handler,
const char *dev_name)
{
return scst_register_virtual_device_node(dev_handler, dev_name,
NUMA_NO_NODE);
}
void scst_unregister_virtual_device(int id);
/*
scst/include/scst_sgv.h
+10
View File
@@ -71,6 +71,16 @@ enum sgv_clustering_types {
sgv_full_clustering,
};
struct sgv_pool *sgv_pool_create_node(const char *name,
enum sgv_clustering_types clustered, int single_alloc_pages,
bool shared, int purge_interval, int nodeid);
static inline struct sgv_pool *sgv_pool_create(const char *name,
enum sgv_clustering_types clustered, int single_alloc_pages,
bool shared, int purge_interval)
{
return sgv_pool_create_node(name, clustered, single_alloc_pages,
shared, purge_interval, NUMA_NO_NODE);
}
struct sgv_pool *sgv_pool_create(const char *name,
enum sgv_clustering_types clustered, int single_alloc_pages,
bool shared, int purge_interval);
scst/src/dev_handlers/scst_vdisk.c
+66 -13
View File
@@ -248,6 +248,7 @@ struct scst_vdisk_dev {
/* Only to pass it to attach() callback. Don't use them anywhere else! */
int blk_shift;
int numa_node_id;
enum scst_dif_mode dif_mode;
int dif_type;
__be64 dif_static_app_tag_combined;
@@ -730,6 +731,7 @@ static struct scst_dev_type vdisk_file_devtype = {
.add_device_parameters =
"blocksize, "
"filename, "
"numa_node_id, "
"nv_cache, "
"o_direct, "
"cluster_mode, "
@@ -789,6 +791,7 @@ static struct scst_dev_type vdisk_blk_devtype = {
"dif_static_app_tag, "
"dif_filename, "
"filename, "
"numa_node_id, "
"nv_cache, "
"cluster_mode, "
"read_only, "
@@ -833,10 +836,11 @@ static struct scst_dev_type vdisk_null_devtype = {
.dev_attrs = vdisk_nullio_attrs,
.add_device_parameters =
"blocksize, "
"dummy, "
"dif_mode, "
"dif_type, "
"dif_static_app_tag, ",
"dummy, "
"numa_node_id, "
"cluster_mode, "
"read_only, "
"removable, "
@@ -7770,8 +7774,8 @@ static void vdev_inq_changed_fn(struct work_struct *work)
}
/* scst_vdisk_mutex supposed to be held */
static int vdev_create(struct scst_dev_type *devt,
const char *name, struct scst_vdisk_dev **res_virt_dev)
static int vdev_create_node(struct scst_dev_type *devt,
const char *name, int nodeid, struct scst_vdisk_dev **res_virt_dev)
{
int res;
struct scst_vdisk_dev *virt_dev, *vv;
@@ -7782,7 +7786,7 @@ static int vdev_create(struct scst_dev_type *devt,
goto out;
/* It's read-mostly, so cache alignment isn't needed */
virt_dev = kzalloc(sizeof(*virt_dev), GFP_KERNEL);
virt_dev = kzalloc_node(sizeof(*virt_dev), GFP_KERNEL, nodeid);
if (virt_dev == NULL) {
PRINT_ERROR("Allocation of virtual device %s failed",
devt->name);
@@ -7805,6 +7809,7 @@ static int vdev_create(struct scst_dev_type *devt,
INIT_WORK(&virt_dev->vdev_inq_changed_work, vdev_inq_changed_fn);
virt_dev->blk_shift = DEF_DISK_BLOCK_SHIFT;
virt_dev->numa_node_id = NUMA_NO_NODE;
if (strlen(name) >= sizeof(virt_dev->name)) {
PRINT_ERROR("Name %s is too long (max allowed %zd)", name,
@@ -7862,6 +7867,12 @@ out_free:
goto out;
}
static inline int vdev_create(struct scst_dev_type *devt,
const char *name, struct scst_vdisk_dev **res_virt_dev)
{
return vdev_create_node(devt, name, NUMA_NO_NODE, res_virt_dev);
}
static void vdev_destroy(struct scst_vdisk_dev *virt_dev)
{
cancel_work_sync(&virt_dev->vdev_inq_changed_work);
@@ -7875,6 +7886,33 @@ static void vdev_destroy(struct scst_vdisk_dev *virt_dev)
return;
}
static void vdev_check_node(struct scst_vdisk_dev **pvirt_dev, int orig_nodeid)
{
struct scst_vdisk_dev *virt_dev = *pvirt_dev;
int nodeid = virt_dev->numa_node_id;
TRACE_ENTRY();
if (virt_dev->numa_node_id != orig_nodeid) {
struct scst_vdisk_dev *v;
TRACE_MEM("Realloc virt_dev %s on node %d", virt_dev->name, nodeid);
/* It's read-mostly, so cache alignment isn't needed */
v = kzalloc_node(sizeof(*v), GFP_KERNEL, nodeid);
if (v == NULL) {
PRINT_ERROR("Reallocation of virtual device %s failed",
virt_dev->name);
goto out;
}
*v = *virt_dev;
kfree(virt_dev);
*pvirt_dev = v;
}
out:
TRACE_EXIT();
return;
}
#ifndef CONFIG_SCST_PROC
static int vdev_parse_add_dev_params(struct scst_vdisk_dev *virt_dev,
@@ -8083,6 +8121,14 @@ static int vdev_parse_add_dev_params(struct scst_vdisk_dev *virt_dev,
}
TRACE_DBG("block size %lld, block shift %d",
val, virt_dev->blk_shift);
} else if (!strcasecmp("numa_node_id", p)) {
virt_dev->numa_node_id = val;
BUILD_BUG_ON(NUMA_NO_NODE != -1);
if (virt_dev->numa_node_id < NUMA_NO_NODE) {
res = -EINVAL;
goto out;
}
TRACE_DBG("numa_node_id %d", virt_dev->numa_node_id);
} else if (!strcasecmp("dif_type", p)) {
virt_dev->dif_type = val;
TRACE_DBG("DIF type %d", virt_dev->dif_type);
@@ -8144,12 +8190,14 @@ static int vdev_fileio_add_device(const char *device_name, char *params)
goto out_destroy;
}
vdev_check_node(&virt_dev, NUMA_NO_NODE);
list_add_tail(&virt_dev->vdev_list_entry, &vdev_list);
vdisk_report_registering(virt_dev);
virt_dev->virt_id = scst_register_virtual_device(virt_dev->vdev_devt,
virt_dev->name);
virt_dev->virt_id = scst_register_virtual_device_node(virt_dev->vdev_devt,
virt_dev->name, virt_dev->numa_node_id);
if (virt_dev->virt_id < 0) {
res = virt_dev->virt_id;
goto out_del;
@@ -8178,7 +8226,8 @@ static int vdev_blockio_add_device(const char *device_name, char *params)
"removable", "blocksize", "nv_cache",
"rotational", "cluster_mode",
"thin_provisioned", "tst",
"dif_mode", "dif_type", "dif_static_app_tag",
"numa_node_id", "dif_mode",
"dif_type", "dif_static_app_tag",
"dif_filename", NULL };
struct scst_vdisk_dev *virt_dev;
@@ -8205,6 +8254,8 @@ static int vdev_blockio_add_device(const char *device_name, char *params)
goto out_destroy;
}
vdev_check_node(&virt_dev, NUMA_NO_NODE);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30)
res = vdisk_create_bioset(virt_dev);
if (res != 0)
@@ -8215,8 +8266,8 @@ static int vdev_blockio_add_device(const char *device_name, char *params)
vdisk_report_registering(virt_dev);
virt_dev->virt_id = scst_register_virtual_device(virt_dev->vdev_devt,
virt_dev->name);
virt_dev->virt_id = scst_register_virtual_device_node(virt_dev->vdev_devt,
virt_dev->name, virt_dev->numa_node_id);
if (virt_dev->virt_id < 0) {
res = virt_dev->virt_id;
goto out_del;
@@ -8243,8 +8294,8 @@ static int vdev_nullio_add_device(const char *device_name, char *params)
int res = 0;
static const char *const allowed_params[] = {
"read_only", "dummy", "removable", "blocksize", "rotational",
"cluster_mode", "dif_mode", "dif_type", "dif_static_app_tag",
"size", "size_mb", "tst", NULL
"size", "size_mb", "tst", "numa_node_id",
"cluster_mode", "dif_mode", "dif_type", "dif_static_app_tag", NULL
};
struct scst_vdisk_dev *virt_dev;
@@ -8263,12 +8314,14 @@ static int vdev_nullio_add_device(const char *device_name, char *params)
if (res != 0)
goto out_destroy;
vdev_check_node(&virt_dev, NUMA_NO_NODE);
list_add_tail(&virt_dev->vdev_list_entry, &vdev_list);
vdisk_report_registering(virt_dev);
virt_dev->virt_id = scst_register_virtual_device(virt_dev->vdev_devt,
virt_dev->name);
virt_dev->virt_id = scst_register_virtual_device_node(virt_dev->vdev_devt,
virt_dev->name, virt_dev->numa_node_id);
if (virt_dev->virt_id < 0) {
res = virt_dev->virt_id;
goto out_del;
scst/src/scst_lib.c
+38 -13
View File
@@ -4162,19 +4162,21 @@ static int scst_dif_none_type1(struct scst_cmd *cmd);
#endif
/* Called under scst_mutex and suspended activity */
int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev)
int scst_alloc_device(gfp_t gfp_mask, int nodeid,
struct scst_device **out_dev)
{
struct scst_device *dev;
int res = 0;
TRACE_ENTRY();
dev = kmem_cache_zalloc(scst_dev_cachep, gfp_mask);
dev = kmem_cache_alloc_node(scst_dev_cachep, gfp_mask, nodeid);
if (dev == NULL) {
PRINT_ERROR("%s", "Allocation of scst_device failed");
res = -ENOMEM;
goto out;
}
memset(dev, 0, sizeof(*dev));
dev->handler = &scst_null_devtype;
#ifdef CONFIG_SCST_PER_DEVICE_CMD_COUNT_LIMIT
@@ -4194,6 +4196,7 @@ int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev)
#endif
dev->dev_double_ua_possible = 1;
dev->queue_alg = SCST_QUEUE_ALG_1_UNRESTRICTED_REORDER;
dev->dev_numa_node_id = nodeid;
scst_pr_init(dev);
@@ -5053,7 +5056,8 @@ int scst_tgt_dev_setup_threads(struct scst_tgt_dev *tgt_dev)
tgt_dev->dev->virt_name);
} else {
/* Create new context */
aic_keeper = kzalloc(sizeof(*aic_keeper), GFP_KERNEL);
aic_keeper = kzalloc_node(sizeof(*aic_keeper), GFP_KERNEL,
dev->dev_numa_node_id);
if (aic_keeper == NULL) {
PRINT_ERROR("Unable to alloc aic_keeper "
"(size %zd)", sizeof(*aic_keeper));
@@ -5260,6 +5264,16 @@ static int scst_alloc_add_tgt_dev(struct scst_session *sess,
else
clear_bit(SCST_TGT_DEV_BLACK_HOLE, &tgt_dev->tgt_dev_flags);
#ifdef CONFIG_CPUMASK_OFFSTACK
tgt_dev->pools = kzalloc_node(sizeof(tgt_dev->pools[0])*NR_CPUS,
GFP_KERNEL, dev->dev_numa_node_id);
if (tgt_dev->pools == NULL) {
PRINT_ERROR("Unable to alloc tgt_dev->pools (size %zd)",
sizeof(tgt_dev->pools[0])*NR_CPUS);
goto out_free;
}
#endif
scst_sgv_pool_use_norm(tgt_dev);
if (dev->scsi_dev != NULL) {
@@ -5317,7 +5331,7 @@ static int scst_alloc_add_tgt_dev(struct scst_session *sess,
"Persistent Reservations", sess->tgt->tgtt->name,
dev->virt_name);
res = -EPERM;
goto out_free;
goto out_free_ua;
}
dev->not_pr_supporting_tgt_devs_num++;
}
@@ -5378,8 +5392,13 @@ out_dec_free:
if (tgtt->get_initiator_port_transport_id == NULL)
dev->not_pr_supporting_tgt_devs_num--;
out_free:
out_free_ua:
scst_free_all_UA(tgt_dev);
#ifdef CONFIG_CPUMASK_OFFSTACK
kfree(tgt_dev->pools);
out_free:
#endif
kmem_cache_free(scst_tgtd_cachep, tgt_dev);
goto out;
}
@@ -5437,6 +5456,10 @@ static void scst_free_tgt_dev(struct scst_tgt_dev *tgt_dev)
scst_tgt_dev_stop_threads(tgt_dev);
#ifdef CONFIG_CPUMASK_OFFSTACK
kfree(tgt_dev->pools);
#endif
kmem_cache_free(scst_tgtd_cachep, tgt_dev);
TRACE_EXIT();
@@ -5983,7 +6006,7 @@ static int scst_ws_push_single_write(struct scst_write_same_priv *wsp,
dif_bufflen = blocks << SCST_DIF_TAG_SHIFT;
cmd->expected_transfer_len_full += dif_bufflen;
dif_sg = sgv_pool_alloc(ws_cmd->tgt_dev->pool,
dif_sg = sgv_pool_alloc(ws_cmd->tgt_dev->pools[raw_smp_processor_id()],
dif_bufflen, GFP_KERNEL, 0, &dif_sg_cnt, &dif_sgv,
&cmd->dev->dev_mem_lim, NULL);
if (unlikely(dif_sg == NULL)) {
@@ -7490,8 +7513,9 @@ int scst_alloc_space(struct scst_cmd *cmd)
if (cmd->no_sgv)
flags |= SGV_POOL_ALLOC_NO_CACHED;
cmd->sg = sgv_pool_alloc(tgt_dev->pool, cmd->bufflen, gfp_mask, flags,
&cmd->sg_cnt, &cmd->sgv, &cmd->dev->dev_mem_lim, NULL);
cmd->sg = sgv_pool_alloc(tgt_dev->pools[raw_smp_processor_id()],
cmd->bufflen, gfp_mask, flags, &cmd->sg_cnt, &cmd->sgv,
&cmd->dev->dev_mem_lim, NULL);
if (unlikely(cmd->sg == NULL))
goto out;
@@ -7510,8 +7534,9 @@ int scst_alloc_space(struct scst_cmd *cmd)
else
dif_bufflen = cmd->bufflen;
cmd->dif_sg = sgv_pool_alloc(tgt_dev->pool, dif_bufflen, gfp_mask, flags,
&cmd->dif_sg_cnt, &cmd->dif_sgv, &cmd->dev->dev_mem_lim, NULL);
cmd->dif_sg = sgv_pool_alloc(tgt_dev->pools[raw_smp_processor_id()],
dif_bufflen, gfp_mask, flags, &cmd->dif_sg_cnt, &cmd->dif_sgv,
&cmd->dev->dev_mem_lim, NULL);
if (unlikely(cmd->dif_sg == NULL))
goto out_sg_free;
@@ -7538,9 +7563,9 @@ int scst_alloc_space(struct scst_cmd *cmd)
if (cmd->data_direction != SCST_DATA_BIDI)
goto success;
cmd->out_sg = sgv_pool_alloc(tgt_dev->pool, cmd->out_bufflen, gfp_mask,
flags, &cmd->out_sg_cnt, &cmd->out_sgv,
&cmd->dev->dev_mem_lim, NULL);
cmd->out_sg = sgv_pool_alloc(tgt_dev->pools[raw_smp_processor_id()],
cmd->out_bufflen, gfp_mask, flags, &cmd->out_sg_cnt,
&cmd->out_sgv, &cmd->dev->dev_mem_lim, NULL);
if (unlikely(cmd->out_sg == NULL))
goto out_dif_sg_free;
scst/src/scst_main.c
+21 -16
View File
@@ -1154,7 +1154,7 @@ static int scst_register_device(struct scsi_device *scsidp)
goto out;
#endif
res = scst_alloc_device(GFP_KERNEL, &dev);
res = scst_alloc_device(GFP_KERNEL, NUMA_NO_NODE, &dev);
if (res != 0)
goto out_unlock;
@@ -1400,16 +1400,17 @@ out:
}
/**
* scst_register_virtual_device() - register a virtual device.
* scst_register_virtual_device_node() - register a virtual device.
* @dev_handler: the device's device handler
* @dev_name: the new device name, NULL-terminated string. Must be uniq
* among all virtual devices in the system.
* @nodeid: NUMA node id this device belongs to or NUMA_NO_NODE.
*
* Registers a virtual device and returns ID assigned to the device on
* success, or negative value otherwise
*/
int scst_register_virtual_device(struct scst_dev_type *dev_handler,
const char *dev_name)
int scst_register_virtual_device_node(struct scst_dev_type *dev_handler,
const char *dev_name, int nodeid)
{
int res;
struct scst_device *dev, *d;
@@ -1446,7 +1447,7 @@ int scst_register_virtual_device(struct scst_dev_type *dev_handler,
if (res != 0)
goto out_resume;
res = scst_alloc_device(GFP_KERNEL, &dev);
res = scst_alloc_device(GFP_KERNEL, nodeid, &dev);
if (res != 0)
goto out_unlock;
@@ -1550,7 +1551,7 @@ out_resume:
scst_resume_activity();
goto out;
}
EXPORT_SYMBOL_GPL(scst_register_virtual_device);
EXPORT_SYMBOL_GPL(scst_register_virtual_device_node);
/**
* scst_unregister_virtual_device() - unegister a virtual device.
@@ -1897,7 +1898,7 @@ int scst_add_threads(struct scst_cmd_threads *cmd_threads,
{
int res = 0, i;
struct scst_cmd_thread_t *thr;
int n = 0, tgt_dev_num = 0;
int n = 0, tgt_dev_num = 0, nodeid = NUMA_NO_NODE;
TRACE_ENTRY();
@@ -1911,7 +1912,7 @@ int scst_add_threads(struct scst_cmd_threads *cmd_threads,
spin_unlock(&cmd_threads->thr_lock);
TRACE_DBG("cmd_threads %p, dev %s, tgt_dev %p, num %d, n %d",
cmd_threads, dev ? dev->virt_name : NULL, tgt_dev, num, n);
cmd_threads, dev ? dev->virt_name : "NULL", tgt_dev, num, n);
if (tgt_dev != NULL) {
struct scst_tgt_dev *t;
@@ -1922,29 +1923,33 @@ int scst_add_threads(struct scst_cmd_threads *cmd_threads,
break;
tgt_dev_num++;
}
}
nodeid = tgt_dev->dev->dev_numa_node_id;
} else if (dev != NULL)
nodeid = dev->dev_numa_node_id;
for (i = 0; i < num; i++) {
thr = kmem_cache_zalloc(scst_thr_cachep, GFP_KERNEL);
thr = kmem_cache_alloc_node(scst_thr_cachep, GFP_KERNEL, nodeid);
if (!thr) {
res = -ENOMEM;
PRINT_ERROR("Fail to allocate thr %d", res);
goto out_wait;
}
memset(thr, 0, sizeof(*thr));
INIT_LIST_HEAD(&thr->thr_active_cmd_list);
spin_lock_init(&thr->thr_cmd_list_lock);
thr->thr_cmd_threads = cmd_threads;
if (dev != NULL) {
thr->cmd_thread = kthread_create(scst_cmd_thread,
thr, "%.13s%d", dev->virt_name, n++);
thr->cmd_thread = kthread_create_on_node(scst_cmd_thread,
thr, nodeid, "%.13s%d", dev->virt_name, n++);
} else if (tgt_dev != NULL) {
thr->cmd_thread = kthread_create(scst_cmd_thread,
thr, "%.10s%d_%d",
thr->cmd_thread = kthread_create_on_node(scst_cmd_thread,
thr, nodeid, "%.10s%d_%d",
tgt_dev->dev->virt_name, tgt_dev_num, n++);
} else
thr->cmd_thread = kthread_create(scst_cmd_thread,
thr, "scstd%d", n++);
thr->cmd_thread = kthread_create_on_node(scst_cmd_thread,
thr, nodeid, "scstd%d", n++);
if (IS_ERR(thr->cmd_thread)) {
res = PTR_ERR(thr->cmd_thread);
scst/src/scst_mem.c
+130 -124
View File
@@ -40,6 +40,12 @@
/* Max pages freed from a pool per shrinking iteration */
#define MAX_PAGES_PER_POOL 50
bool scst_force_global_sgv_pool;
static struct sgv_pool *sgv_dma_pool_per_cpu[NR_CPUS];
static struct sgv_pool *sgv_norm_clust_pool_per_cpu[NR_CPUS];
static struct sgv_pool *sgv_norm_pool_per_cpu[NR_CPUS];
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
#if defined(CONFIG_LOCKDEP) && !defined(CONFIG_SCST_PROC)
static struct lock_class_key scst_pool_key;
@@ -61,10 +67,6 @@ static int sgv_max_local_pages, sgv_max_trans_pages;
static DEFINE_SPINLOCK(sgv_pools_lock); /* inner lock for sgv_pool_lock! */
static DEFINE_MUTEX(sgv_pools_mutex);
/* Both protected by sgv_pools_lock */
static struct sgv_pool *sgv_cur_purge_pool;
static LIST_HEAD(sgv_active_pools_list);
static atomic_t sgv_releases_on_hiwmk = ATOMIC_INIT(0);
static atomic_t sgv_releases_on_hiwmk_failed = ATOMIC_INIT(0);
@@ -97,24 +99,39 @@ static inline bool sgv_pool_clustered(const struct sgv_pool *pool)
void scst_sgv_pool_use_norm(struct scst_tgt_dev *tgt_dev)
{
int i;
tgt_dev->tgt_dev_gfp_mask = __GFP_NOWARN;
tgt_dev->pool = sgv_norm_pool;
for (i = 0; i < NR_CPUS; i++)
if (!scst_force_global_sgv_pool)
tgt_dev->pools[i] = sgv_norm_pool_per_cpu[i];
else
tgt_dev->pools[i] = sgv_norm_pool;
tgt_dev->tgt_dev_clust_pool = 0;
}
void scst_sgv_pool_use_norm_clust(struct scst_tgt_dev *tgt_dev)
{
int i;
TRACE_MEM("%s", "Use clustering");
tgt_dev->tgt_dev_gfp_mask = __GFP_NOWARN;
tgt_dev->pool = sgv_norm_clust_pool;
for (i = 0; i < NR_CPUS; i++)
if (!scst_force_global_sgv_pool)
tgt_dev->pools[i] = sgv_norm_clust_pool_per_cpu[i];
else
tgt_dev->pools[i] = sgv_norm_clust_pool;
tgt_dev->tgt_dev_clust_pool = 1;
}
void scst_sgv_pool_use_dma(struct scst_tgt_dev *tgt_dev)
{
int i;
TRACE_MEM("%s", "Use ISA DMA memory");
tgt_dev->tgt_dev_gfp_mask = __GFP_NOWARN | GFP_DMA;
tgt_dev->pool = sgv_dma_pool;
for (i = 0; i < NR_CPUS; i++)
if (!scst_force_global_sgv_pool)
tgt_dev->pools[i] = sgv_dma_pool_per_cpu[i];
else
tgt_dev->pools[i] = sgv_dma_pool;
tgt_dev->tgt_dev_clust_pool = 0;
}
@@ -143,49 +160,11 @@ static void sgv_dtor_and_free(struct sgv_pool_obj *obj)
return;
}
/* Might be called under sgv_pool_lock */
static inline void sgv_del_from_active(struct sgv_pool *pool)
{
struct list_head *next;
TRACE_MEM("Deleting sgv pool %p from the active list", pool);
spin_lock_bh(&sgv_pools_lock);
next = pool->sgv_active_pools_list_entry.next;
list_del(&pool->sgv_active_pools_list_entry);
if (sgv_cur_purge_pool == pool) {
TRACE_MEM("Sgv pool %p is sgv cur purge pool", pool);
if (next == &sgv_active_pools_list)
next = next->next;
if (next == &sgv_active_pools_list) {
sgv_cur_purge_pool = NULL;
TRACE_MEM("%s", "Sgv active list now empty");
} else {
sgv_cur_purge_pool = list_entry(next, typeof(*pool),
sgv_active_pools_list_entry);
TRACE_MEM("New sgv cur purge pool %p",
sgv_cur_purge_pool);
}
}
spin_unlock_bh(&sgv_pools_lock);
return;
}
/* Must be called under sgv_pool_lock held */
static void sgv_dec_cached_entries(struct sgv_pool *pool, int pages)
{
pool->cached_entries--;
pool->cached_pages -= pages;
if (pool->cached_entries == 0)
sgv_del_from_active(pool);
return;
}
/* Must be called under sgv_pool_lock held */
@@ -286,68 +265,28 @@ static int __sgv_shrink(int nr, int min_interval, int *out_freed)
{
struct sgv_pool *pool;
unsigned long cur_time = jiffies;
int prev_nr = nr;
bool circle = false;
int prev_nr = nr + 1;
TRACE_ENTRY();
TRACE_MEM("Trying to shrink %d pages from all sgv pools "
"(min_interval %d)", nr, min_interval);
while (nr > 0) {
struct list_head *next;
while (prev_nr > nr && nr > 0) {
prev_nr = nr;
spin_lock_bh(&sgv_pools_lock);
pool = sgv_cur_purge_pool;
if (pool == NULL) {
if (list_empty(&sgv_active_pools_list)) {
TRACE_MEM("%s", "Active pools list is empty");
goto out_unlock;
}
pool = list_first_entry(&sgv_active_pools_list,
typeof(*pool),
sgv_active_pools_list_entry);
}
sgv_pool_get(pool);
next = pool->sgv_active_pools_list_entry.next;
if (next == &sgv_active_pools_list) {
if (circle && (prev_nr == nr)) {
TRACE_MEM("Full circle done, but no progress, "
"leaving (nr %d)", nr);
goto out_unlock_put;
}
circle = true;
prev_nr = nr;
next = next->next;
}
sgv_cur_purge_pool = list_entry(next, typeof(*pool),
sgv_active_pools_list_entry);
TRACE_MEM("New cur purge pool %p", sgv_cur_purge_pool);
spin_unlock_bh(&sgv_pools_lock);
nr = sgv_shrink_pool(pool, nr, min_interval, cur_time, out_freed);
sgv_pool_put(pool);
mutex_lock(&sgv_pools_mutex);
list_for_each_entry(pool, &sgv_pools_list,
sgv_pools_list_entry) {
if (pool->cached_entries)
nr = sgv_shrink_pool(pool, nr, min_interval,
cur_time, out_freed);
}
mutex_unlock(&sgv_pools_mutex);
}
out:
TRACE_EXIT_RES(nr);
return nr;
out_unlock:
spin_unlock_bh(&sgv_pools_lock);
goto out;
out_unlock_put:
spin_unlock_bh(&sgv_pools_lock);
sgv_pool_put(pool);
goto out;
}
static unsigned long __sgv_can_be_shrunk(void)
@@ -359,8 +298,7 @@ static unsigned long __sgv_can_be_shrunk(void)
TRACE_ENTRY();
spin_lock_bh(&sgv_pools_lock);
list_for_each_entry(pool, &sgv_active_pools_list,
sgv_active_pools_list_entry) {
list_for_each_entry(pool, &sgv_pools_list, sgv_pools_list_entry) {
if (pool->purge_interval > 0)
inactive_pages += pool->inactive_cached_pages;
}
@@ -774,14 +712,6 @@ static struct sgv_pool_obj *sgv_get_obj(struct sgv_pool *pool, int cache_num,
}
get_new:
if (pool->cached_entries == 0) {
TRACE_MEM("Adding pool %p to the active list", pool);
spin_lock_bh(&sgv_pools_lock);
list_add_tail(&pool->sgv_active_pools_list_entry,
&sgv_active_pools_list);
spin_unlock_bh(&sgv_pools_lock);
}
pool->cached_entries++;
pool->cached_pages += pages;
@@ -1390,7 +1320,8 @@ void scst_free_sg(struct scatterlist *sg, int count)
EXPORT_SYMBOL_GPL(scst_free_sg);
/* Must be called under sgv_pools_mutex */
static void sgv_pool_init_cache(struct sgv_pool *pool, int cache_num)
static void sgv_pool_init_cache(struct sgv_pool *pool, int cache_num,
bool per_cpu)
{
int size;
int pages;
@@ -1423,14 +1354,15 @@ static void sgv_pool_init_cache(struct sgv_pool *pool, int cache_num)
/* both sgv and trans_tbl are kmalloc'ed() */
}
TRACE_MEM("pages=%d, size=%d", pages, size);
TRACE_MEM("pages=%d, size=%d (per cpu %d)", pages, size, per_cpu);
scnprintf(pool->cache_names[cache_num],
sizeof(pool->cache_names[cache_num]),
"%s-%uK", pool->name, (pages << PAGE_SHIFT) >> 10);
pool->caches[cache_num] = kmem_cache_create(
pool->cache_names[cache_num], size,
0, SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN, NULL
0, per_cpu ? SCST_SLAB_FLAGS :
(SCST_SLAB_FLAGS|SLAB_HWCACHE_ALIGN), NULL
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
, NULL);
#else
@@ -1442,7 +1374,7 @@ static void sgv_pool_init_cache(struct sgv_pool *pool, int cache_num)
/* Must be called under sgv_pools_mutex */
static int sgv_pool_init(struct sgv_pool *pool, const char *name,
enum sgv_clustering_types clustering_type, int single_alloc_pages,
int purge_interval)
int purge_interval, bool per_cpu)
{
int res = -ENOMEM;
int i;
@@ -1495,7 +1427,7 @@ static int sgv_pool_init(struct sgv_pool *pool, const char *name,
pool->owner_mm = current->mm;
for (i = 0; i < pool->max_caches; i++) {
sgv_pool_init_cache(pool, i);
sgv_pool_init_cache(pool, i, per_cpu);
if (pool->caches[i] == NULL) {
PRINT_ERROR("Allocation of sgv_pool "
"cache %s(%d) failed", name, i);
@@ -1653,7 +1585,7 @@ void sgv_pool_set_allocator(struct sgv_pool *pool,
EXPORT_SYMBOL_GPL(sgv_pool_set_allocator);
/**
* sgv_pool_create - creates and initializes an SGV pool
* sgv_pool_create_node - creates and initializes an SGV pool
* @name: the name of the SGV pool
* @clustered: sets type of the pages clustering.
* @single_alloc_pages: if 0, then the SGV pool will work in the set of
@@ -1663,7 +1595,7 @@ EXPORT_SYMBOL_GPL(sgv_pool_set_allocator);
* @shared: sets if the SGV pool can be shared between devices or not.
* The cache sharing allowed only between devices created inside
* the same address space. If an SGV pool is shared, each
* subsequent call of sgv_pool_create() with the same cache name
* subsequent call of sgv_pool_create*() with the same cache name
* will not create a new cache, but instead return a reference
* to it.
* @purge_interval: sets the cache purging interval. I.e., an SG buffer
@@ -1671,20 +1603,27 @@ EXPORT_SYMBOL_GPL(sgv_pool_set_allocator);
* purge_interval <= t < 2*purge_interval. If purge_interval
* is 0, then the default interval will be used (60 seconds).
* If purge_interval <0, then the automatic purging will be
* disabled.
* disabled. In HZ.
* @nodeid: NUMA node for this pool. Can be NUMA_NO_NODE, if the
* caller doesn't care.
*
* Description:
* Returns the resulting SGV pool or NULL in case of any error.
*/
struct sgv_pool *sgv_pool_create(const char *name,
struct sgv_pool *sgv_pool_create_node(const char *name,
enum sgv_clustering_types clustering_type,
int single_alloc_pages, bool shared, int purge_interval)
int single_alloc_pages, bool shared, int purge_interval, int nodeid)
{
struct sgv_pool *pool;
int rc;
TRACE_ENTRY();
TRACE_MEM("Creating pool %s (clustering_type %d, "
"single_alloc_pages %d, shared %d, purge_interval %d, "
"nodeid %d)", name, clustering_type, single_alloc_pages,
shared, purge_interval, nodeid);
mutex_lock(&sgv_pools_mutex);
list_for_each_entry(pool, &sgv_pools_list, sgv_pools_list_entry) {
@@ -1706,15 +1645,16 @@ struct sgv_pool *sgv_pool_create(const char *name,
}
}
pool = kmem_cache_zalloc(sgv_pool_cachep, GFP_KERNEL);
pool = kmem_cache_alloc_node(sgv_pool_cachep, GFP_KERNEL, nodeid);
if (pool == NULL) {
PRINT_ERROR("Allocation of sgv_pool failed (size %zd)",
sizeof(*pool));
goto out_unlock;
}
memset(pool, 0, sizeof(*pool));
rc = sgv_pool_init(pool, name, clustering_type, single_alloc_pages,
purge_interval);
purge_interval, nodeid != NUMA_NO_NODE);
if (rc != 0)
goto out_free;
@@ -1729,7 +1669,7 @@ out_free:
pool = NULL;
goto out_unlock;
}
EXPORT_SYMBOL_GPL(sgv_pool_create);
EXPORT_SYMBOL_GPL(sgv_pool_create_node);
/**
* sgv_pool_get - increase ref counter for the corresponding SGV pool
@@ -1783,7 +1723,7 @@ EXPORT_SYMBOL_GPL(sgv_pool_del);
/* Both parameters in pages */
int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark)
{
int res = 0;
int res = 0, i;
TRACE_ENTRY();
@@ -1810,6 +1750,44 @@ int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark)
if (sgv_dma_pool == NULL)
goto out_free_clust;
/*
* ToDo: not compatible with CPU hotplug! Notification
* callbacks must be installed!
*/
for (i = 0; i < NR_CPUS; i++) {
char name[60];
if (!cpu_online(i))
continue;
scnprintf(name, sizeof(name), "sgv-%d", i);
sgv_norm_pool_per_cpu[i] = sgv_pool_create_node(name,
sgv_no_clustering, 0, false, 0, cpu_to_node(i));
if (sgv_norm_pool_per_cpu[i] == NULL)
goto out_free_per_cpu_norm;
}
for (i = 0; i < NR_CPUS; i++) {
char name[60];
if (!cpu_online(i))
continue;
scnprintf(name, sizeof(name), "sgv-clust-%d", i);
sgv_norm_clust_pool_per_cpu[i] = sgv_pool_create_node(name,
sgv_full_clustering, 0, false, 0, cpu_to_node(i));
if (sgv_norm_clust_pool_per_cpu[i] == NULL)
goto out_free_per_cpu_clust;
}
for (i = 0; i < NR_CPUS; i++) {
char name[60];
if (!cpu_online(i))
continue;
scnprintf(name, sizeof(name), "sgv-dma-%d", i);
sgv_dma_pool_per_cpu[i] = sgv_pool_create_node(name,
sgv_no_clustering, 0, false, 0, cpu_to_node(i));
if (sgv_dma_pool_per_cpu[i] == NULL)
goto out_free_per_cpu_dma;
}
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
sgv_shrinker = set_shrinker(DEFAULT_SEEKS, sgv_shrink);
#else
@@ -1827,6 +1805,23 @@ out:
TRACE_EXIT_RES(res);
return res;
out_free_per_cpu_dma:
for (i = 0; i < NR_CPUS; i++)
if (sgv_dma_pool_per_cpu[i] != NULL)
sgv_pool_destroy(sgv_dma_pool_per_cpu[i]);
out_free_per_cpu_clust:
for (i = 0; i < NR_CPUS; i++)
if (sgv_norm_clust_pool_per_cpu[i] != NULL)
sgv_pool_destroy(sgv_norm_clust_pool_per_cpu[i]);
out_free_per_cpu_norm:
for (i = 0; i < NR_CPUS; i++)
if (sgv_norm_pool_per_cpu[i] != NULL)
sgv_pool_destroy(sgv_norm_pool_per_cpu[i]);
sgv_pool_destroy(sgv_dma_pool);
out_free_clust:
sgv_pool_destroy(sgv_norm_clust_pool);
@@ -1843,6 +1838,8 @@ out_err:
void scst_sgv_pools_deinit(void)
{
int i;
TRACE_ENTRY();
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23))
@@ -1852,8 +1849,19 @@ void scst_sgv_pools_deinit(void)
#endif
sgv_pool_destroy(sgv_dma_pool);
for (i = 0; i < NR_CPUS; i++)
if (sgv_dma_pool_per_cpu[i] != NULL)
sgv_pool_destroy(sgv_dma_pool_per_cpu[i]);
sgv_pool_destroy(sgv_norm_pool);
for (i = 0; i < NR_CPUS; i++)
if (sgv_norm_pool_per_cpu[i] != NULL)
sgv_pool_destroy(sgv_norm_pool_per_cpu[i]);
sgv_pool_destroy(sgv_norm_clust_pool);
for (i = 0; i < NR_CPUS; i++)
if (sgv_norm_clust_pool_per_cpu[i] != NULL)
sgv_pool_destroy(sgv_norm_clust_pool_per_cpu[i]);
kmem_cache_destroy(sgv_pool_cachep);
@@ -1925,8 +1933,7 @@ int sgv_procinfo_show(struct seq_file *seq, void *v)
TRACE_ENTRY();
spin_lock_bh(&sgv_pools_lock);
list_for_each_entry(pool, &sgv_active_pools_list,
sgv_active_pools_list_entry) {
list_for_each_entry(pool, &sgv_pools_list, sgv_pools_list_entry) {
inactive_pages += pool->inactive_cached_pages;
}
spin_unlock_bh(&sgv_pools_lock);
@@ -2051,8 +2058,7 @@ static ssize_t sgv_sysfs_global_stat_show(struct kobject *kobj,
TRACE_ENTRY();
spin_lock_bh(&sgv_pools_lock);
list_for_each_entry(pool, &sgv_active_pools_list,
sgv_active_pools_list_entry) {
list_for_each_entry(pool, &sgv_pools_list, sgv_pools_list_entry) {
inactive_pages += pool->inactive_cached_pages;
}
spin_unlock_bh(&sgv_pools_lock);
scst/src/scst_mem.h
+2 -2
View File
@@ -114,8 +114,6 @@ struct sgv_pool {
struct work_struct sgv_purge_work;
#endif
struct list_head sgv_active_pools_list_entry;
atomic_t big_alloc, big_pages, big_merged;
atomic_t other_alloc, other_pages, other_merged;
@@ -137,6 +135,8 @@ struct sgv_pool {
struct completion *sgv_kobj_release_cmpl;
};
extern bool scst_force_global_sgv_pool;
static inline struct scatterlist *sgv_pool_sg(struct sgv_pool_obj *obj)
{
return obj->sg_entries;
scst/src/scst_priv.h
+1 -1
View File
@@ -353,7 +353,7 @@ void scst_queue_retry_cmd(struct scst_cmd *cmd);
int scst_alloc_tgt(struct scst_tgt_template *tgtt, struct scst_tgt **tgt);
void scst_free_tgt(struct scst_tgt *tgt);
int scst_alloc_device(gfp_t gfp_mask, struct scst_device **out_dev);
int scst_alloc_device(gfp_t gfp_mask, int nodeid, struct scst_device **out_dev);
void scst_free_device(struct scst_device *dev);
bool scst_device_is_exported(struct scst_device *dev);
scst/src/scst_sysfs.c
+102
View File
@@ -32,6 +32,7 @@
#endif
#include "scst_priv.h"
#include "scst_pres.h"
#include "scst_mem.h"
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
#ifdef CONFIG_LOCKDEP
@@ -3537,6 +3538,69 @@ static struct kobj_attribute dev_max_tgt_dev_commands_attr =
scst_dev_sysfs_max_tgt_dev_commands_show,
scst_dev_sysfs_max_tgt_dev_commands_store);
static ssize_t scst_dev_numa_node_id_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
int pos = 0;
struct scst_device *dev;
TRACE_ENTRY();
dev = container_of(kobj, struct scst_device, dev_kobj);
pos = sprintf(buf, "%d\n%s", dev->dev_numa_node_id,
(dev->dev_numa_node_id != NUMA_NO_NODE) ?
SCST_SYSFS_KEY_MARK "\n" : "");
TRACE_EXIT_RES(pos);
return pos;
}
static ssize_t scst_dev_numa_node_id_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
int res;
struct scst_device *dev;
long newtn;
TRACE_ENTRY();
dev = container_of(kobj, struct scst_device, dev_kobj);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
res = kstrtol(buf, 0, &newtn);
#else
res = strict_strtol(buf, 0, &newtn);
#endif
if (res != 0) {
PRINT_ERROR("strtol() for %s failed: %d ", buf, res);
goto out;
}
BUILD_BUG_ON(NUMA_NO_NODE != -1);
if (newtn < NUMA_NO_NODE) {
PRINT_ERROR("Illegal numa_node_id value %ld", newtn);
res = -EINVAL;
goto out;
}
if (dev->dev_numa_node_id != newtn) {
PRINT_INFO("Setting new NUMA node id %ld for device %s (old %d)",
newtn, dev->virt_name, dev->dev_numa_node_id);
dev->dev_numa_node_id = newtn;
}
out:
if (res == 0)
res = count;
TRACE_EXIT_RES(res);
return res;
}
static struct kobj_attribute dev_numa_node_id_attr =
__ATTR(numa_node_id, S_IRUGO | S_IWUSR, scst_dev_numa_node_id_show,
scst_dev_numa_node_id_store);
static ssize_t scst_dev_block_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
@@ -3668,6 +3732,7 @@ static struct kobj_attribute dev_block_attr =
static struct attribute *scst_dev_attrs[] = {
&dev_type_attr.attr,
&dev_max_tgt_dev_commands_attr.attr,
&dev_numa_node_id_attr.attr,
&dev_block_attr.attr,
NULL,
};
@@ -7287,6 +7352,42 @@ static struct kobj_attribute scst_main_trace_level_attr =
#endif /* defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING) */
static ssize_t scst_force_global_sgv_pool_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
return sprintf(buf, "%d\n%s\n", scst_force_global_sgv_pool,
scst_force_global_sgv_pool ? SCST_SYSFS_KEY_MARK "\n": "");
}
static ssize_t scst_force_global_sgv_pool_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
int res;
unsigned long v;
TRACE_ENTRY();
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
res = kstrtoul(buf, 0, &v);
#else
res = strict_strtoul(buf, 0, &v);
#endif
if (res)
goto out;
scst_force_global_sgv_pool = v;
res = count;
out:
TRACE_EXIT_RES(res);
return res;
}
static struct kobj_attribute scst_force_global_sgv_pool_attr =
__ATTR(force_global_sgv_pool, S_IRUGO | S_IWUSR,
scst_force_global_sgv_pool_show, scst_force_global_sgv_pool_store);
static void __printf(2, 3) scst_append(void *arg, const char *fmt, ...)
{
char *buf = arg;
@@ -7483,6 +7584,7 @@ static struct attribute *scst_sysfs_root_default_attrs[] = {
#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
&scst_main_trace_level_attr.attr,
#endif
&scst_force_global_sgv_pool_attr.attr,
&scst_trace_cmds_attr.attr,
&scst_trace_mcmds_attr.attr,
&scst_version_attr.attr,