diff --git a/srpt/README b/srpt/README index b051f835f..72a5929db 100644 --- a/srpt/README +++ b/srpt/README @@ -380,12 +380,29 @@ Performance Notes - Target Side Performance Notes - Initiator Side ---------------------------------- +* Using multiple RDMA connections between initator and target results in a + significant performance improvement. To benefit from this feature, use + kernel 3.19 or later at the initiator side and enable scsi-mq either by + setting SCSI_MQ_DEFAULT=y in the kernel config or via the following command: + + echo Y > /sys/module/scsi_mod/parameters/use_blk_mq + + If the HCA model in your initiator system supports multiple MSI-X interrupts + the next step is either to stop the irqbalance service or to write a policy + script that stops irqbalance from modifying the IB interrupt CPU + affinity. Once this has been done spread the IB interrupts uniformly over + CPU cores via e.g. scripts/spread-mlx4-ib-interrupts. + + For more information about scsi-mq see also Michael Larabel, SCSI + Multi-Queue Performance Appears Great For Linux 3.17, Phoronix, June 18, + 2014 (http://www.phoronix.com/scan.php?page=news_item&px=MTcyMjk). + * Choose a proper value for the ib_srp kernel module parameter cmd_sg_entries. The default value 12 works well for buffered reads while the throughput for write-dominated workloads improves by changing this value into 255. One way to set this kernel module parameter is as follows: - echo options ib_srp cmd_sg_entries=255 >>/etc/modprobe.d/ib_srp.conf + echo options ib_srp cmd_sg_entries=255 >/etc/modprobe.d/ib_srp.conf * For multithreaded workloads using small block sizes changing rq_affinity into 2 improves IOPS significantly (Linux kernel 3.1 and later; see also diff --git a/srpt/src/ib_srpt.c b/srpt/src/ib_srpt.c index 63a861e36..9cbfeb835 100644 --- a/srpt/src/ib_srpt.c +++ b/srpt/src/ib_srpt.c @@ -92,7 +92,6 @@ MODULE_LICENSE("Dual BSD/GPL"); */ static u64 srpt_service_guid; -/* List of srpt_device structures. */ static atomic_t srpt_device_count; #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING) static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS; @@ -200,7 +199,7 @@ static struct scst_tgt_template srpt_template; static void srpt_unregister_mad_agent(struct srpt_device *sdev); #ifdef CONFIG_SCST_PROC static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt); -#endif /*CONFIG_SCST_PROC*/ +#endif /* CONFIG_SCST_PROC */ static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch, struct srpt_send_ioctx *ioctx); static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch); @@ -2043,12 +2042,13 @@ static void srpt_process_send_completion(struct ib_cq *cq, srpt_send_context); } else if (opcode == SRPT_RDMA_READ_LAST || opcode == SRPT_RDMA_WRITE_LAST) { - PRINT_INFO("RDMA t %d for idx %u failed with status %d." - "%s", opcode, index, wc->status, + PRINT_INFO("RDMA t %d for idx %u failed with status %d.%s", + opcode, index, wc->status, + wc->status == IB_WC_RETRY_EXC_ERR ? + " If this has not been triggered by a cable pull, please consider to increase the subnet timeout parameter on the IB switch." : wc->status == IB_WC_WR_FLUSH_ERR ? - " If this has not been triggered by a cable" - " pull, please check the involved IB HCA's" - " and cables." : ""); + " If this has not been triggered by a cable pull, please check the involved IB HCA's and cables." : + ""); srpt_handle_rdma_err_comp(ch, ch->ioctx_ring[index], opcode, srpt_xmt_rsp_context); } else if (opcode == SRPT_RDMA_ZEROLENGTH_WRITE) { @@ -2496,7 +2496,7 @@ static bool srpt_is_target_enabled(struct scst_tgt *scst_tgt) return srpt_tgt && srpt_tgt->enabled; } -#endif +#endif /* CONFIG_SCST_PROC */ /* * srpt_next_comp_vector() - Next completion vector >= srpt_tgt->comp_vector @@ -4199,7 +4199,7 @@ static const struct attribute *srpt_sess_attrs[] = { #endif NULL }; -#endif +#endif /* CONFIG_SCST_PROC */ /* SCST target template for the SRP target implementation. */ static struct scst_tgt_template srpt_template = { @@ -4250,7 +4250,7 @@ static struct scst_proc_data srpt_log_proc_data = { }; #endif -#endif /*CONFIG_SCST_PROC*/ +#endif /* CONFIG_SCST_PROC */ /* Note: the caller must have zero-initialized *@srpt_tgt. */ static void srpt_init_tgt(struct srpt_tgt *srpt_tgt) @@ -4596,7 +4596,7 @@ static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt) #endif } -#endif /*CONFIG_SCST_PROC*/ +#endif /* CONFIG_SCST_PROC */ /** * srpt_init_module() - Kernel module initialization. @@ -4706,14 +4706,14 @@ static int __init srpt_init_module(void) PRINT_ERROR("couldn't register procfs entry"); goto out_rdma_cm; } -#endif /*CONFIG_SCST_PROC*/ +#endif /* CONFIG_SCST_PROC */ return 0; #ifdef CONFIG_SCST_PROC out_rdma_cm: rdma_destroy_id(rdma_cm_id); -#endif /*CONFIG_SCST_PROC*/ +#endif /* CONFIG_SCST_PROC */ out_unregister_client: ib_unregister_client(&srpt_client); out_unregister_target: @@ -4731,7 +4731,7 @@ static void __exit srpt_cleanup_module(void) ib_unregister_client(&srpt_client); #ifdef CONFIG_SCST_PROC srpt_unregister_procfs_entry(&srpt_template); -#endif /*CONFIG_SCST_PROC*/ +#endif /* CONFIG_SCST_PROC */ scst_unregister_target_template(&srpt_template); TRACE_EXIT(); @@ -4739,10 +4739,3 @@ static void __exit srpt_cleanup_module(void) module_init(srpt_init_module); module_exit(srpt_cleanup_module); - -/* - * Local variables: - * c-basic-offset: 8 - * indent-tabs-mode: t - * End: - */ diff --git a/srpt/src/ib_srpt.h b/srpt/src/ib_srpt.h index 30abefbeb..7d78a7d47 100644 --- a/srpt/src/ib_srpt.h +++ b/srpt/src/ib_srpt.h @@ -175,10 +175,10 @@ static inline u32 idx_from_wr_id(u64 wr_id) } struct rdma_iu { - u64 raddr; - u32 rkey; - struct ib_sge *sge; - u32 sge_cnt; + u64 raddr; + u32 rkey; + struct ib_sge *sge; + u32 sge_cnt; }; /** @@ -501,10 +501,3 @@ struct srp_login_req_rdma { }; #endif /* IB_SRPT_H */ - -/* - * Local variables: - * c-basic-offset: 8 - * indent-tabs-mode: t - * End: - */