iscsi-scst: add per_sess_dedicated_tgt_threads attribute in ini groups

If this attrubute is set, each iSCSI session for this initiator has
dedicated, i.e. not shared with other sessions, pool of the iscsi{wr,rd}
kernel threads.

Useful to control per-session CPU affinity to improve performance.



git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@7069 d57e44dd-8a1f-0410-8b47-8ef2f437770f

iscsi-scst/README

@@ -371,6 +371,13 @@ Each connection subdirectory contains the following entries:
 
  - state - contains processing state of this connection.
 
+Each initiator group subdirectory contains:
+
+ - per_sess_dedicated_tgt_threads - if set, each iSCSI session has
+   dedicated, i.e. not shared with other sessions, pool of the
+   iscsi{wr,rd} kernel threads. Useful to control per-session CPU
+   affinity to improve performance. Default: not set.
+
 See SCST README for info about other attributes.
 
 Below is a sample script, which configures 1 virtual disk "disk1" using

iscsi-scst/README_in-tree

@@ -200,6 +200,13 @@ Each connection subdirectory contains the following entries:
 
  - state - contains processing state of this connection.
 
+Each initiator group subdirectory contains:
+
+ - per_sess_dedicated_tgt_threads - if set, each iSCSI session has
+   dedicated, i.e. not shared with other sessions, pool of the
+   iscsi{wr,rd} kernel threads. Useful to control per-session CPU
+   affinity to improve performance. Default: not set.
+
 See SCST README for info about other attributes.
 
 Below is a sample script, which configures 1 virtual disk "disk1" using

iscsi-scst/kernel/iscsi.c

@@ -4091,6 +4091,7 @@ struct scst_tgt_template iscsi_template = {
 	.tgtt_attrs = iscsi_attrs,
 	.tgt_attrs = iscsi_tgt_attrs,
 	.sess_attrs = iscsi_sess_attrs,
+	.acg_attrs = iscsi_acg_attrs,
 	.enable_target = iscsi_enable_target,
 	.is_target_enabled = iscsi_is_target_enabled,
 	.add_target = iscsi_sysfs_add_target,
@@ -4192,7 +4193,7 @@ void iscsi_threads_pool_put(struct iscsi_thread_pool *p)
 	return;
 }
 
-int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
+int iscsi_threads_pool_get(bool dedicated, const cpumask_t *cpu_mask,
 	struct iscsi_thread_pool **out_pool)
 {
 	int res;
@@ -4205,9 +4206,16 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
 
 	mutex_lock(&iscsi_threads_pool_mutex);
 
+	if (dedicated) {
+		/* Ignore cpu_mask, if set */
+		cpu_mask = NULL;
+		goto create;
+	}
+
 	list_for_each_entry(p, &iscsi_thread_pools_list,
 			thread_pools_list_entry) {
-		if (!cpu_mask || cpumask_equal(cpu_mask, &p->cpu_mask)) {
+		if ((!cpu_mask || cpumask_equal(cpu_mask, &p->cpu_mask)) &&
+		    !p->dedicated) {
 			p->thread_pool_ref++;
 			TRACE_DBG("iSCSI thread pool %p found (new ref %d)",
 				p, p->thread_pool_ref);
@@ -4216,7 +4224,9 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
 		}
 	}
 
-	TRACE_DBG("%s", "Creating new iSCSI thread pool");
+create:
+	TRACE_DBG("Creating new iSCSI thread pool (dedicated %d, cpu_mask %p)",
+		dedicated, cpu_mask);
 
 	p = kmem_cache_zalloc(iscsi_thread_pool_cache, GFP_KERNEL);
 	if (p == NULL) {
@@ -4247,8 +4257,11 @@ int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
 	p->thread_pool_ref = 1;
 	mutex_init(&p->tp_mutex);
 	INIT_LIST_HEAD(&p->threads_list);
+	p->dedicated = dedicated;
 
-	if (cpu_mask == NULL)
+	if (dedicated)
+		count = 1;
+	else if (cpu_mask == NULL)
 		count = max_t(int, num_online_cpus(), 2);
 	else {
 		count = 0;
@@ -4395,7 +4408,7 @@ static int __init iscsi_init(void)
 	iscsi_conn_ktype.sysfs_ops = scst_sysfs_get_sysfs_ops();
 #endif
 
-	err = iscsi_threads_pool_get(NULL, &iscsi_main_thread_pool);
+	err = iscsi_threads_pool_get(false, NULL, &iscsi_main_thread_pool);
 	if (err != 0)
 		goto out_thr;
 

iscsi-scst/kernel/iscsi.h

@@ -87,6 +87,7 @@ struct iscsi_thread_pool {
 	wait_queue_head_t wr_waitQ;
 
 	cpumask_t cpu_mask;
+	bool dedicated;
 
 	int thread_pool_ref;
 
@@ -558,7 +559,7 @@ extern int iscsi_preliminary_complete(struct iscsi_cmnd *req,
 	struct iscsi_cmnd *orig_req, bool get_data);
 extern int set_scst_preliminary_status_rsp(struct iscsi_cmnd *req,
 	bool get_data, int key, int asc, int ascq);
-extern int iscsi_threads_pool_get(const cpumask_t *cpu_mask,
+extern int iscsi_threads_pool_get(bool dedicated, const cpumask_t *cpu_mask,
 	struct iscsi_thread_pool **out_pool);
 extern void iscsi_threads_pool_put(struct iscsi_thread_pool *p);
 
@@ -633,6 +634,7 @@ extern void __iscsi_del_attr(struct iscsi_target *target,
 /* session.c */
 #ifndef CONFIG_SCST_PROC
 extern const struct attribute *iscsi_sess_attrs[];
+extern const struct attribute *iscsi_acg_attrs[];
 #endif
 extern const struct file_operations session_seq_fops;
 extern struct iscsi_session *session_lookup(struct iscsi_target *, u64);

iscsi-scst/kernel/session.c

@@ -99,7 +99,9 @@ static int iscsi_session_alloc(struct iscsi_target *target,
 	}
 
 	if (!session->sess_params.rdma_extensions) {
-		err = iscsi_threads_pool_get(&session->scst_sess->acg->acg_cpu_mask,
+		err = iscsi_threads_pool_get(
+			(bool)scst_get_acg_tgt_priv(session->scst_sess->acg),
+			&session->scst_sess->acg->acg_cpu_mask,
 			&session->sess_thr_pool);
 		if (err != 0)
 			goto err_unreg;
@@ -289,6 +291,8 @@ out_err_unlock:
 
 static void __session_free(struct iscsi_session *session)
 {
+	if (session->sess_thr_pool)
+		iscsi_threads_pool_put(session->sess_thr_pool);
 	kfree(session->initiator_name);
 	kmem_cache_free(iscsi_sess_cache, session);
 }
@@ -347,11 +351,6 @@ int session_free(struct iscsi_session *session, bool del)
 	if (del)
 		list_del(&session->session_list_entry);
 
-	if (session->sess_thr_pool != NULL) {
-		iscsi_threads_pool_put(session->sess_thr_pool);
-		session->sess_thr_pool = NULL;
-	}
-
 	if (session->scst_sess != NULL) {
 		/*
 		 * We must NOT call scst_unregister_session() in the waiting

iscsi-scst/kernel/target.c

@@ -655,4 +655,63 @@ ssize_t iscsi_sysfs_mgmt_cmd(char *cmd)
 	return res;
 }
 
+static ssize_t iscsi_acg_sess_dedicated_threads_show(struct kobject *kobj,
+	struct kobj_attribute *attr, char *buf)
+{
+	int pos;
+	struct scst_acg *acg;
+	bool dedicated;
+
+	TRACE_ENTRY();
+
+	acg = container_of(kobj, struct scst_acg, acg_kobj);
+	dedicated = (bool)scst_get_acg_tgt_priv(acg);
+
+	pos = sprintf(buf, "%d\n%s", dedicated,
+		dedicated ? SCST_SYSFS_KEY_MARK "\n" : "");
+
+	TRACE_EXIT_RES(pos);
+	return pos;
+}
+
+static ssize_t iscsi_acg_sess_dedicated_threads_store(struct kobject *kobj,
+	struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	int res;
+	struct scst_acg *acg;
+	unsigned long val;
+
+	TRACE_ENTRY();
+
+	acg = container_of(kobj, struct scst_acg, acg_kobj);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
+	res = kstrtoul(buf, 0, &val);
+#else
+	res = strict_strtoul(buf, 0, &val);
+#endif
+	if (res != 0) {
+		PRINT_ERROR("strict_strtoul() for %s failed: %d ", buf, res);
+		goto out;
+	}
+
+	scst_set_acg_tgt_priv(acg, (void *)(unsigned long)(val != 0));
+
+	res = count;
+
+out:
+	TRACE_EXIT_RES(res);
+	return res;
+}
+
+static struct kobj_attribute iscsi_acg_attr_sess_dedicated_threads =
+	__ATTR(per_sess_dedicated_tgt_threads, S_IRUGO | S_IWUSR,
+		iscsi_acg_sess_dedicated_threads_show,
+		iscsi_acg_sess_dedicated_threads_store);
+
+const struct attribute *iscsi_acg_attrs[] = {
+	&iscsi_acg_attr_sess_dedicated_threads.attr,
+	NULL,
+};
+
 #endif /* CONFIG_SCST_PROC */

scst/include/scst.h

@@ -1239,6 +1239,9 @@ struct scst_tgt_template {
 
 	/* sysfs session attributes, if any */
 	const struct attribute **sess_attrs;
+
+	/* sysfs ACG attributes, if any */
+	const struct attribute **acg_attrs;
 #endif
 
 	/* Optional help string for mgmt_cmd commands */
@@ -3342,7 +3345,11 @@ struct scst_acg {
 	struct kobject *initiators_kobj;
 #endif
 
+	/* LUNS addressing method for all LUNs in this ACG */
 	enum scst_lun_addr_method addr_method;
+
+	/* Private stuff for target drivers */
+	void *acg_tgt_priv;
 };
 
 /*
@@ -4875,6 +4882,19 @@ static inline void scst_set_aen_delivery_status(struct scst_aen *aen,
 	aen->delivery_status = status;
 }
 
+/*
+ * Get/Set functions for tgt's target private data
+ */
+static inline void *scst_get_acg_tgt_priv(struct scst_acg *acg)
+{
+	return acg->acg_tgt_priv;
+}
+
+static inline void scst_set_acg_tgt_priv(struct scst_acg *acg, void *val)
+{
+	acg->acg_tgt_priv = val;
+}
+
 void scst_aen_done(struct scst_aen *aen);
 
 static inline struct scatterlist *__sg_next_inline(struct scatterlist *sg)

scst/src/scst_sysfs.c

@@ -5581,6 +5581,16 @@ int scst_acg_sysfs_create(struct scst_tgt *tgt,
 		goto out_del;
 	}
 
+	if (acg->tgt->tgtt->acg_attrs) {
+		res = sysfs_create_files(&acg->acg_kobj,
+					 acg->tgt->tgtt->acg_attrs);
+		if (res != 0) {
+			PRINT_ERROR("Can't add attributes for acg %s",
+				acg->acg_name);
+			goto out_del;
+		}
+	}
+
 out:
 	TRACE_EXIT_RES(res);
 	return res;