ib_srpt: Merge r5990:6028 from trunk

git-svn-id: http://svn.code.sf.net/p/scst/svn/branches/iser@6030 d57e44dd-8a1f-0410-8b47-8ef2f437770f

srpt/README

@@ -380,12 +380,29 @@ Performance Notes - Target Side
 Performance Notes - Initiator Side
 ----------------------------------
 
+* Using multiple RDMA connections between initator and target results in a
+  significant performance improvement. To benefit from this feature, use
+  kernel 3.19 or later at the initiator side and enable scsi-mq either by
+  setting SCSI_MQ_DEFAULT=y in the kernel config or via the following command:
+
+  echo Y > /sys/module/scsi_mod/parameters/use_blk_mq
+
+  If the HCA model in your initiator system supports multiple MSI-X interrupts
+  the next step is either to stop the irqbalance service or to write a policy
+  script that stops irqbalance from modifying the IB interrupt CPU
+  affinity. Once this has been done spread the IB interrupts uniformly over
+  CPU cores via e.g. scripts/spread-mlx4-ib-interrupts.
+
+  For more information about scsi-mq see also Michael Larabel, SCSI
+  Multi-Queue Performance Appears Great For Linux 3.17, Phoronix, June 18,
+  2014 (http://www.phoronix.com/scan.php?page=news_item&px=MTcyMjk).
+
 * Choose a proper value for the ib_srp kernel module parameter
   cmd_sg_entries. The default value 12 works well for buffered reads while
   the throughput for write-dominated workloads improves by changing this value
   into 255. One way to set this kernel module parameter is as follows:
 
-  echo options ib_srp cmd_sg_entries=255 >>/etc/modprobe.d/ib_srp.conf
+  echo options ib_srp cmd_sg_entries=255 >/etc/modprobe.d/ib_srp.conf
 
 * For multithreaded workloads using small block sizes changing rq_affinity
   into 2 improves IOPS significantly (Linux kernel 3.1 and later; see also

srpt/src/ib_srpt.c

@@ -92,7 +92,6 @@ MODULE_LICENSE("Dual BSD/GPL");
  */
 
 static u64 srpt_service_guid;
-/* List of srpt_device structures. */
 static atomic_t srpt_device_count;
 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
@@ -200,7 +199,7 @@ static struct scst_tgt_template srpt_template;
 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
 #ifdef CONFIG_SCST_PROC
 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
-#endif /*CONFIG_SCST_PROC*/
+#endif /* CONFIG_SCST_PROC */
 static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
 				    struct srpt_send_ioctx *ioctx);
 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch);
@@ -2043,12 +2042,13 @@ static void srpt_process_send_completion(struct ib_cq *cq,
 						  srpt_send_context);
 		} else if (opcode == SRPT_RDMA_READ_LAST ||
 			   opcode == SRPT_RDMA_WRITE_LAST) {
-			PRINT_INFO("RDMA t %d for idx %u failed with status %d."
-				   "%s", opcode, index, wc->status,
+			PRINT_INFO("RDMA t %d for idx %u failed with status %d.%s",
+				   opcode, index, wc->status,
+				   wc->status == IB_WC_RETRY_EXC_ERR ?
+				   " If this has not been triggered by a cable pull, please consider to increase the subnet timeout parameter on the IB switch." :
 				   wc->status == IB_WC_WR_FLUSH_ERR ?
-				   " If this has not been triggered by a cable"
-				   " pull, please check the involved IB HCA's"
-				   " and cables." : "");
+				   " If this has not been triggered by a cable pull, please check the involved IB HCA's and cables." :
+				   "");
 			srpt_handle_rdma_err_comp(ch, ch->ioctx_ring[index],
 						  opcode, srpt_xmt_rsp_context);
 		} else if (opcode == SRPT_RDMA_ZEROLENGTH_WRITE) {
@@ -2496,7 +2496,7 @@ static bool srpt_is_target_enabled(struct scst_tgt *scst_tgt)
 
 	return srpt_tgt && srpt_tgt->enabled;
 }
-#endif
+#endif /* CONFIG_SCST_PROC */
 
 /*
  * srpt_next_comp_vector() - Next completion vector >= srpt_tgt->comp_vector
@@ -4199,7 +4199,7 @@ static const struct attribute *srpt_sess_attrs[] = {
 #endif
 	NULL
 };
-#endif
+#endif /* CONFIG_SCST_PROC */
 
 /* SCST target template for the SRP target implementation. */
 static struct scst_tgt_template srpt_template = {
@@ -4250,7 +4250,7 @@ static struct scst_proc_data srpt_log_proc_data = {
 };
 #endif
 
-#endif /*CONFIG_SCST_PROC*/
+#endif /* CONFIG_SCST_PROC */
 
 /* Note: the caller must have zero-initialized *@srpt_tgt. */
 static void srpt_init_tgt(struct srpt_tgt *srpt_tgt)
@@ -4596,7 +4596,7 @@ static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
 #endif
 }
 
-#endif /*CONFIG_SCST_PROC*/
+#endif /* CONFIG_SCST_PROC */
 
 /**
  * srpt_init_module() - Kernel module initialization.
@@ -4706,14 +4706,14 @@ static int __init srpt_init_module(void)
 		PRINT_ERROR("couldn't register procfs entry");
 		goto out_rdma_cm;
 	}
-#endif /*CONFIG_SCST_PROC*/
+#endif /* CONFIG_SCST_PROC */
 
 	return 0;
 
 #ifdef CONFIG_SCST_PROC
 out_rdma_cm:
 	rdma_destroy_id(rdma_cm_id);
-#endif /*CONFIG_SCST_PROC*/
+#endif /* CONFIG_SCST_PROC */
 out_unregister_client:
 	ib_unregister_client(&srpt_client);
 out_unregister_target:
@@ -4731,7 +4731,7 @@ static void __exit srpt_cleanup_module(void)
 	ib_unregister_client(&srpt_client);
 #ifdef CONFIG_SCST_PROC
 	srpt_unregister_procfs_entry(&srpt_template);
-#endif /*CONFIG_SCST_PROC*/
+#endif /* CONFIG_SCST_PROC */
 	scst_unregister_target_template(&srpt_template);
 
 	TRACE_EXIT();
@@ -4739,10 +4739,3 @@ static void __exit srpt_cleanup_module(void)
 
 module_init(srpt_init_module);
 module_exit(srpt_cleanup_module);
-
-/*
- * Local variables:
- * c-basic-offset:   8
- * indent-tabs-mode: t
- * End:
- */

srpt/src/ib_srpt.h

@@ -175,10 +175,10 @@ static inline u32 idx_from_wr_id(u64 wr_id)
 }
 
 struct rdma_iu {
-	u64 raddr;
-	u32 rkey;
-	struct ib_sge *sge;
-	u32 sge_cnt;
+	u64		raddr;
+	u32		rkey;
+	struct ib_sge	*sge;
+	u32		sge_cnt;
 };
 
 /**
@@ -501,10 +501,3 @@ struct srp_login_req_rdma {
 };
 
 #endif				/* IB_SRPT_H */
-
-/*
- * Local variables:
- * c-basic-offset:   8
- * indent-tabs-mode: t
- * End:
- */