diff --git a/debian/copyright b/debian/copyright index 35ccb1244..6a8e1152c 100644 --- a/debian/copyright +++ b/debian/copyright @@ -131,15 +131,6 @@ Files: iscsi-scst/AskingQuestions iscsi-scst/kernel/Kconfig iscsi-scst/kernel/Makefile.in-kernel iscsi-scst/kernel/iscsit_transport.c - iscsi-scst/kernel/patches/put_page_callback-2.6.16.29.patch - iscsi-scst/kernel/patches/put_page_callback-2.6.18.1.patch - iscsi-scst/kernel/patches/put_page_callback-2.6.21.1.patch - iscsi-scst/kernel/patches/put_page_callback-2.6.22.patch - iscsi-scst/kernel/patches/put_page_callback-2.6.23.patch - iscsi-scst/kernel/patches/put_page_callback-2.6.24.patch - iscsi-scst/kernel/patches/put_page_callback-2.6.25.patch - iscsi-scst/kernel/patches/rhel/put_page_callback-2.6.18.patch - iscsi-scst/kernel/patches/rhel/put_page_callback-rhel5.patch iscsi-scst/usr/sha1.h Copyright: 2007-2017, Vladislav Bolkhovitin License: GPL-2 diff --git a/iscsi-scst/README b/iscsi-scst/README index 37bd611dd..01756e6e0 100644 --- a/iscsi-scst/README +++ b/iscsi-scst/README @@ -73,49 +73,6 @@ source iSCSI targets, when for data transmission: data will be additionally copied into temporary TCP buffers. The performance hit will be quite noticeable. -Note, that if your network hardware does not support TX offload -functions or has them disabled, then TCP zero-copy transmit functions on -your system will not be used by Linux networking in any case, so -put_page_callback patch will not be able to improve performance for you. -You can check your network hardware offload capabilities by command -"ethtool -k ethX", where X is the network device number. At least -"tx-checksumming" and "scatter-gather" should be enabled. - -If you have in your kernel log error messages like: - -iscsi-scst: ***ERROR*** net_priv isn't NULL and != ref_cmd - -with the corresponding kernel BUG dump, then put_page_callback patch you -use isn't sufficient for your kernel. This might be because the kernel -you use has some additional patches applied, which affect the -functionality, which put_page_callback patch provides. For example, -Fedora or Gentoo use kernels, which, although have version number like -2.6.18, are greatly differ from the "vanilla" kernel 2.6.18, -maintained by Linus Torvalds for that the put_page_callback patch was -created. In this case it is recommended either: - - - Search net/ in your kernel source for "put_page" and "get_page" functions. - If you find any in some place, except in net/sunrpc/svc.c and - net/core/pktgen.c, then, most likely, you found the reason of your - problem. Replace them by "net_put_page" and "net_get_page" - correspondingly and try again. If the problem is solved, then please - prepare a new put_page_callback patch and send it to the SCST mailing - list scst-devel@lists.sourceforge.net. - -or - - - Unapply this patch and use iSCSI-SCST without it. Also report this - problem to the SCST mailing list scst-devel@lists.sourceforge.net. - - -IMPORTANT: This patch does not support compound pages, so not working with -========= network drivers using them. Sings of that is either when - iSCSI-SCST keeps closing connections, because no reply from - initiator for too long time, or system crashes on NULL - pointer dereference in iscsi_get_page_callback. You can find - list of such drivers by searching in drivers/net for - __GFP_COMP word in their source code. - Usage ----- @@ -945,71 +902,6 @@ in/out in the kernel's module Makefile: random places. -Creating version of put_page_callback patch for your kernel ------------------------------------------------------------ - -If you need your own version of put_page_callback patch for your custom -kernel, for which there is no prepared version, you can create it -yourself. This is pretty mechanical work, you don't need to understand -how it works, you only need to do the following two steps: - -1. Apply the closest version of put_page_callback-.patch -on your kernel. Resolve only failed hunks from include/ and -net/core/utils.c, ignore other failures. - -2. Search net/ in your kernel source for "put_page" and "get_page" -functions. Replace them by "net_put_page" and "net_get_page" -correspondingly. - -For vanilla kernels that should be all. Then please send your new -put_page_callback-.patch to the SCST mailing list -scst-devel@lists.sourceforge.net. - -But some out of tree drivers (some versions of DRBD and Intel e1000 are -reported to do so) use own pages referencing on the TX path, so they -should be modified the same way as above as well. (No need to modify RX -path, because put_page_callback logic used by SCST only on the TX path.) - -In case of DRBD, if you don't do this modification, you can specify -disable_sendpage parameter for drbd module. But in this case it might be -better for performance simply not apply put_page_callback patch. - - -Background information about zero-copy data sending ---------------------------------------------------- - -As explained above the most efficient operation of the iSCSI-SCST target -driver is achieved when the following two conditions are met: -* Data is sent from target to initiator in a zero-copy fashion. -* Data buffers are cached for reuse (by the so-called sgv pool). -Unfortunately the zero-copy API in the Linux kernel (proto.sendpage() / -tcp_sendpage()) does not yet support completion notifications. Hence the -put_page_callback patch which adds completion notification support to -tcp_sendpage(). However, since the put_page_callback patch increases the size -of the page structure in the Linux kernel that patch is considered -unacceptable for integration in the mainline kernel [1]. Another approach, -called skb paged fragment destructors, might get merged in the mainline kernel -in the future. Version five of that patch series has been posted in May 2012 -[2]. - -Notes: -* The current implementation of scst_user is such that iSCSI-SCST only sends - data allocated by scst_user in a zero-copy fashion if the put_page_callback - patch has been applied. This holds even if scst_user did not use the sgv - pool to allocate a data buffer. -* Zero-copy sending is only possible with network interface drivers that - support scatter/gather and checksumming (NETIF_F_SG and NETIF_F_ALL_CSUM - respectively). - -References: -[1] James Bottomley, Linux Kernel Mailing List, December 2008, - http://lkml.org/lkml/2008/12/11/213. -[2] Alex Bligh, Rebase Ian Campbell's skb fragment tracking to 3.2, January - 2013, linux-netdev, http://thread.gmane.org/gmane.linux.network/256820/. -[3] Ian Campbell, [PATCH v5 0/9] skb paged fragment destructors, May 3 2012, - linux-netdev, http://thread.gmane.org/gmane.linux.network/229669. - - Credits ------- diff --git a/iscsi-scst/doc/SCST_Gentoo_HOWTO.txt b/iscsi-scst/doc/SCST_Gentoo_HOWTO.txt index b83c80c1b..05ada92f1 100644 --- a/iscsi-scst/doc/SCST_Gentoo_HOWTO.txt +++ b/iscsi-scst/doc/SCST_Gentoo_HOWTO.txt @@ -138,14 +138,6 @@ is the only safe option for VMware ESXi or older windows versions, for best performance with newer windows versions a blocksize of 4096 will be much faster. -The TCP/IP zero-copy transfer completion notification provided by the -put_page_callback patch will only improve performance if your network -adapter supports TX offloading. You can check this by running ethtool -k -eth0 (substitute eth0 for the nic you use), at least "tx-checksumming" -and "scatter-gather" should be enabled. If your adapter is unable to do -so consider upgrading to a good server grade nic, both Intel and -Broadcom make good server grade nics. - If you have an Intel CPU that supports SSE4.2 make sure to add the crc32c-intel module to the kernel, so iSCSI-SCST will do all digest calculations with hardware acceleration. You can find this option in the diff --git a/iscsi-scst/doc/iscsi-scst-howto.txt b/iscsi-scst/doc/iscsi-scst-howto.txt index 68f91b073..2cc640b5b 100644 --- a/iscsi-scst/doc/iscsi-scst-howto.txt +++ b/iscsi-scst/doc/iscsi-scst-howto.txt @@ -18,33 +18,7 @@ command in a shell: svn co https://scst.svn.sourceforge.net/svnroot/scst/trunk scst -While SCST and iSCSI-SCST work fine with an unpatched Linux kernel with -the same or better speed as other open source iSCSI targets, if you want -even better performance you need to patch and rebuild the kernel. Select -the patch according to your kernel version (2.6.38.x in the example -below): - - cd /usr/src/kernels/linux-2.6.38.8 - patch -p1 < $HOME/scst/iscsi-scst/kernel/patches/put_page_callback-2.6.38.patch - make clean - -If you can't find put_page_callback for your kernel, then the patching -is not needed for it, so you can skip this and next (kernel rebuild and -reboot) steps. - -Next, build and install the kernel: - - make && make modules - make modules_install && make install - -For some Linux distributions (e.g. Ubuntu) the above steps do not rebuild the -initial RAM disk. Make sure you regenerate the RAM disk before rebooting. -Please look up the documentation of the update-initrd, update-initramfs or the -mkinitrd command supplied with your distribution. - -Now reboot your system such that the newly built kernel is loaded. - -Once the system is again up and running the next step is to build and install +The next step is to build and install SCST, iSCSI-SCST and scstadmin. With most distros (including RHEL 6) you can do that by running the following command from the SCST directory: diff --git a/iscsi-scst/include/iscsit_transport.h b/iscsi-scst/include/iscsit_transport.h index 64eade892..38603fa96 100644 --- a/iscsi-scst/include/iscsit_transport.h +++ b/iscsi-scst/include/iscsit_transport.h @@ -53,9 +53,7 @@ struct iscsit_transport { void (*iscsit_close_all_portals)(void); -#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) unsigned int need_alloc_write_buf:1; -#endif struct module *owner; const char name[SCST_MAX_NAME]; diff --git a/iscsi-scst/kernel/iscsi.c b/iscsi-scst/kernel/iscsi.c index 3ddedb1f4..fefc2fcc8 100644 --- a/iscsi-scst/kernel/iscsi.c +++ b/iscsi-scst/kernel/iscsi.c @@ -32,15 +32,6 @@ #include "iscsi.h" #include "digest.h" -#if 0 && !defined(GENERATING_UPSTREAM_PATCH) -#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) -#warning Patch put_page_callback-.patch not applied on your \ -kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION \ -config option not set. ISCSI-SCST will be working with not the best \ -performance. Refer README file for details. -#endif -#endif - #define ISCSI_INIT_WRITE_WAKE 0x1 static int ctr_major; @@ -346,12 +337,6 @@ static struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, iscsi_cmnd_init(conn, cmnd, parent); - if (parent == NULL) { -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - atomic_set(&cmnd->net_ref_cnt, 0); -#endif - } - TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd); return cmnd; } @@ -2077,19 +2062,15 @@ static int scsi_cmnd_start(struct iscsi_cmnd *req) scst_cmd_set_expected(scst_cmd, dir, sz); scst_cmd_set_expected_out_transfer_len(scst_cmd, be32_to_cpu(req_hdr->data_length)); -#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) if (conn->transport->need_alloc_write_buf) scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd); -#endif } } else if (req_hdr->flags & ISCSI_CMD_READ) { dir = SCST_DATA_READ; scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length)); -#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) if (conn->transport->need_alloc_write_buf) scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd); -#endif } else if (req_hdr->flags & ISCSI_CMD_WRITE) { dir = SCST_DATA_WRITE; scst_cmd_set_expected(scst_cmd, dir, @@ -2353,10 +2334,6 @@ static void __cmnd_abort(struct iscsi_cmnd *cmnd) cmnd->conn->session->exp_cmd_sn, cmnd->conn, rdt, cmnd->conn->read_cmnd, cmnd->conn->read_state); -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt)); -#endif - /* * Lock to sync with iscsi_check_tm_data_wait_timeouts(), including * CMD_ABORTED bit set. @@ -3344,7 +3321,6 @@ static ssize_t iscsi_tcp_get_initiator_ip(struct iscsi_conn *conn, return pos; } -#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) static int iscsi_alloc_data_buf(struct scst_cmd *cmd) { /* @@ -3358,7 +3334,6 @@ static int iscsi_alloc_data_buf(struct scst_cmd *cmd) scst_cmd_set_no_sgv(cmd); return 1; } -#endif static void iscsi_tcp_preprocessing_done(struct iscsi_cmnd *req) { @@ -4119,9 +4094,7 @@ struct scst_tgt_template iscsi_template = { #endif .release = iscsi_target_release, .xmit_response = iscsi_xmit_response, -#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) .tgt_alloc_data_buf = iscsi_alloc_data_buf, -#endif .preprocessing_done = iscsi_preprocessing_done, .pre_exec = iscsi_pre_exec, .task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done, @@ -4136,9 +4109,7 @@ static struct iscsit_transport iscsi_tcp_transport = { .owner = THIS_MODULE, .name = "iSCSI-TCP", .transport_type = ISCSI_TCP, -#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) .need_alloc_write_buf = 1, -#endif .iscsit_conn_alloc = iscsi_conn_alloc, .iscsit_conn_activate = conn_activate, .iscsit_conn_free = iscsi_tcp_conn_free, @@ -4349,22 +4320,6 @@ static int __init iscsi_init(void) goto out_free_dummy; } -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - err = net_set_get_put_page_callbacks(iscsi_get_page_callback, - iscsi_put_page_callback); - if (err != 0) { - PRINT_INFO("Unable to set page callbacks: %d", err); - goto out_destroy_mempool; - } -#else -#if 0 && !defined(GENERATING_UPSTREAM_PATCH) - PRINT_WARNING("%s", - "CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION " - "not enabled in your kernel. ISCSI-SCST will be working with " - "not the best performance. Refer README file for details."); -#endif -#endif - ctr_major = register_chrdev(0, ctr_name, &ctr_fops); if (ctr_major < 0) { PRINT_ERROR("failed to register the control device %d", @@ -4451,11 +4406,6 @@ out_reg: unregister_chrdev(ctr_major, ctr_name); out_callb: -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - net_set_get_put_page_callbacks(NULL, NULL); - -out_destroy_mempool: -#endif mempool_destroy(iscsi_cmnd_abort_mempool); out_free_dummy: @@ -4485,10 +4435,6 @@ static void __exit iscsi_exit(void) iscsit_unreg_transport(&iscsi_tcp_transport); -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - net_set_get_put_page_callbacks(NULL, NULL); -#endif - mempool_destroy(iscsi_cmnd_abort_mempool); __free_pages(dummy_page, 0); diff --git a/iscsi-scst/kernel/iscsi.h b/iscsi-scst/kernel/iscsi.h index 76a4e23a1..f457dea65 100644 --- a/iscsi-scst/kernel/iscsi.h +++ b/iscsi-scst/kernel/iscsi.h @@ -502,9 +502,6 @@ struct iscsi_cmnd { }; atomic_t ref_cnt; -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - atomic_t net_ref_cnt; -#endif struct iscsi_pdu pdu; @@ -594,10 +591,6 @@ extern void __iscsi_write_space_ready(struct iscsi_conn *conn); /* nthread.c */ extern int iscsi_send(struct iscsi_conn *conn); -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) -extern void iscsi_get_page_callback(struct page *page); -extern void iscsi_put_page_callback(struct page *page); -#endif extern int istrd(void *arg); extern int istwr(void *arg); extern void iscsi_task_mgmt_affected_cmds_done(struct scst_mgmt_cmd *scst_mcmd); diff --git a/iscsi-scst/kernel/nthread.c b/iscsi-scst/kernel/nthread.c index fe2dc74ba..92e47dc62 100644 --- a/iscsi-scst/kernel/nthread.c +++ b/iscsi-scst/kernel/nthread.c @@ -56,118 +56,6 @@ enum tx_state { TX_END, }; -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) -static void iscsi_check_closewait(struct iscsi_conn *conn) -{ - struct iscsi_cmnd *cmnd; - - TRACE_ENTRY(); - - TRACE_CONN_CLOSE_DBG("conn %p, sk_state %d", conn, - conn->sock->sk->sk_state); - - if (conn->sock->sk->sk_state != TCP_CLOSE) { - TRACE_CONN_CLOSE_DBG("conn %p, skipping", conn); - goto out; - } - - /* - * No data are going to be sent, so all queued buffers can be freed - * now. In many cases TCP does that only in close(), but we can't rely - * on user space on calling it. - */ - -again: - spin_lock_bh(&conn->cmd_list_lock); - list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) { - struct iscsi_cmnd *rsp; - - TRACE_CONN_CLOSE_DBG("cmd %p, scst_state %x, " - "r2t_len_to_receive %d, ref_cnt %d, parent_req %p, " - "net_ref_cnt %d, sg %p", cmnd, cmnd->scst_state, - cmnd->r2t_len_to_receive, atomic_read(&cmnd->ref_cnt), - cmnd->parent_req, atomic_read(&cmnd->net_ref_cnt), - cmnd->sg); - - sBUG_ON(cmnd->parent_req != NULL); - - if (cmnd->sg != NULL) { - int i; - - if (cmnd_get_check(cmnd)) - continue; - - /* - * If we don't unlock here, we are risking to get into - * recursive deadlock in cmnd_done() called from cmnd_put() - */ - spin_unlock_bh(&conn->cmd_list_lock); - - for (i = 0; i < cmnd->sg_cnt; i++) { - struct page *page = sg_page(&cmnd->sg[i]); - - TRACE_CONN_CLOSE_DBG("page %p, net_priv %p, " - "_count %d", page, page->net_priv, - atomic_read(&page->_count)); - - if (page->net_priv != NULL) { - while (page->net_priv != NULL) - iscsi_put_page_callback(page); - } - } - cmnd_put(cmnd); - goto again; - } - - list_for_each_entry(rsp, &cmnd->rsp_cmd_list, - rsp_cmd_list_entry) { - TRACE_CONN_CLOSE_DBG(" rsp %p, ref_cnt %d, " - "net_ref_cnt %d, sg %p", - rsp, atomic_read(&rsp->ref_cnt), - atomic_read(&rsp->net_ref_cnt), rsp->sg); - - if ((rsp->sg != cmnd->sg) && (rsp->sg != NULL)) { - int i; - - if (cmnd_get_check(rsp)) - continue; - - /* - * If we don't unlock here, we are risking to - * get into recursive deadlock in cmnd_done() - * called from cmnd_put() - */ - spin_unlock_bh(&conn->cmd_list_lock); - - for (i = 0; i < rsp->sg_cnt; i++) { - struct page *page = - sg_page(&rsp->sg[i]); - TRACE_CONN_CLOSE_DBG( - " page %p, net_priv %p, " - "_count %d", - page, page->net_priv, - atomic_read(&page->_count)); - - if (page->net_priv != NULL) { - while (page->net_priv != NULL) - iscsi_put_page_callback(page); - } - } - cmnd_put(rsp); - goto again; - } - } - } - spin_unlock_bh(&conn->cmd_list_lock); - -out: - TRACE_EXIT(); - return; -} -#else -static inline void iscsi_check_closewait(struct iscsi_conn *conn) {}; -#endif - static void free_pending_commands(struct iscsi_conn *conn) { struct iscsi_session *session = conn->session; @@ -256,9 +144,6 @@ static void free_orphaned_pending_commands(struct iscsi_conn *conn) static void trace_conn_close(struct iscsi_conn *conn) { struct iscsi_cmnd *cmnd; -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - struct iscsi_cmnd *rsp; -#endif #if 0 if (time_after(jiffies, start_waiting + 10*HZ)) @@ -277,45 +162,6 @@ static void trace_conn_close(struct iscsi_conn *conn) cmnd->scst_cmd->state : -1, cmnd->r2t_len_to_receive, atomic_read(&cmnd->ref_cnt), cmnd->pdu.bhs.sn, cmnd->parent_req, cmnd->pending); -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - TRACE_CONN_CLOSE_DBG("net_ref_cnt %d, sg %p", - atomic_read(&cmnd->net_ref_cnt), - cmnd->sg); - if (cmnd->sg != NULL) { - int i; - - for (i = 0; i < cmnd->sg_cnt; i++) { - struct page *page = sg_page(&cmnd->sg[i]); - - TRACE_CONN_CLOSE_DBG("page %p, " - "net_priv %p, _count %d", - page, page->net_priv, - atomic_read(&page->_count)); - } - } - - sBUG_ON(cmnd->parent_req != NULL); - - list_for_each_entry(rsp, &cmnd->rsp_cmd_list, - rsp_cmd_list_entry) { - TRACE_CONN_CLOSE_DBG(" rsp %p, " - "ref_cnt %d, net_ref_cnt %d, sg %p", - rsp, atomic_read(&rsp->ref_cnt), - atomic_read(&rsp->net_ref_cnt), rsp->sg); - if (rsp->sg != cmnd->sg && rsp->sg) { - int i; - - for (i = 0; i < rsp->sg_cnt; i++) { - TRACE_CONN_CLOSE_DBG(" page %p, " - "net_priv %p, _count %d", - sg_page(&rsp->sg[i]), - sg_page(&rsp->sg[i])->net_priv, - atomic_read(&sg_page(&rsp->sg[i])-> - _count)); - } - } - } -#endif /* CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION */ } spin_unlock_bh(&conn->cmd_list_lock); return; @@ -529,8 +375,6 @@ static void close_conn(struct iscsi_conn *conn) if (!conn->session->sess_params.rdma_extensions) { /* It might never be called for being closed conn */ __iscsi_write_space_ready(conn); - - iscsi_check_closewait(conn); } } @@ -1136,80 +980,6 @@ int istrd(void *arg) return 0; } -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) -static inline void __iscsi_get_page_callback(struct iscsi_cmnd *cmd) -{ - int v; - - TRACE_NET_PAGE("cmd %p, new net_ref_cnt %d", - cmd, atomic_read(&cmd->net_ref_cnt)+1); - - v = atomic_inc_return(&cmd->net_ref_cnt); - if (v == 1) { - TRACE_NET_PAGE("getting cmd %p", cmd); - cmnd_get(cmd); - } - return; -} - -void iscsi_get_page_callback(struct page *page) -{ - struct iscsi_cmnd *cmd = page->net_priv; - - TRACE_NET_PAGE("page %p, _count %d", page, - atomic_read(&page->_count)); - - __iscsi_get_page_callback(cmd); - return; -} - -static inline void __iscsi_put_page_callback(struct iscsi_cmnd *cmd) -{ - TRACE_NET_PAGE("cmd %p, new net_ref_cnt %d", cmd, - atomic_read(&cmd->net_ref_cnt)-1); - - if (atomic_dec_and_test(&cmd->net_ref_cnt)) { - int i, sg_cnt = cmd->sg_cnt; - - for (i = 0; i < sg_cnt; i++) { - struct page *page = sg_page(&cmd->sg[i]); - - TRACE_NET_PAGE("Clearing page %p", page); - if (page->net_priv == cmd) - page->net_priv = NULL; - } - cmnd_put(cmd); - } - return; -} - -void iscsi_put_page_callback(struct page *page) -{ - struct iscsi_cmnd *cmd = page->net_priv; - - TRACE_NET_PAGE("page %p, _count %d", page, - atomic_read(&page->_count)); - - __iscsi_put_page_callback(cmd); - return; -} - -static void check_net_priv(struct iscsi_cmnd *cmd, struct page *page) -{ - smp_rmb(); /* to sync with __iscsi_get_page_callback() */ - if ((atomic_read(&cmd->net_ref_cnt) == 1) && (page->net_priv == cmd)) { - TRACE_DBG("sendpage() not called get_page(), zeroing net_priv " - "%p (page %p)", page->net_priv, page); - page->net_priv = NULL; - } - return; -} -#else -static inline void check_net_priv(struct iscsi_cmnd *cmd, struct page *page) {} -static inline void __iscsi_get_page_callback(struct iscsi_cmnd *cmd) {} -static inline void __iscsi_put_page_callback(struct iscsi_cmnd *cmd) {} -#endif - void req_add_to_write_timeout_list(struct iscsi_cmnd *req) { struct iscsi_conn *conn; @@ -1341,7 +1111,7 @@ static int write_data(struct iscsi_conn *conn) int saved_size, size, sendsize; int length, offset, idx; int flags, res, count, sg_size; - bool do_put = false, ref_cmd_to_parent; + bool ref_cmd_to_parent; TRACE_ENTRY(); @@ -1415,21 +1185,13 @@ retry: goto out; } - /* To protect from too early transfer completion race */ - __iscsi_get_page_callback(ref_cmd); - do_put = true; - sock = conn->sock; -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - sock_sendpage = sock->ops->sendpage; -#else if ((write_cmnd->parent_req->scst_cmd != NULL) && scst_cmd_get_dh_data_buff_alloced(write_cmnd->parent_req->scst_cmd)) sock_sendpage = sock_no_sendpage; else sock_sendpage = sock->ops->sendpage; -#endif flags = MSG_DONTWAIT; sg_size = size; @@ -1471,34 +1233,6 @@ retry: while (1) { sendpage = sock_sendpage; -#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION) - { - static DEFINE_SPINLOCK(net_priv_lock); - - spin_lock(&net_priv_lock); - if (unlikely(page->net_priv != NULL)) { - if (page->net_priv != ref_cmd) { - /* - * This might happen if user space - * supplies to scst_user the same - * pages in different commands or in - * case of zero-copy FILEIO, when - * several initiators request the same - * data simultaneously. - */ - TRACE_DBG("net_priv isn't NULL and != " - "ref_cmd (write_cmnd %p, ref_cmd " - "%p, sg %p, idx %d, page %p, " - "net_priv %p)", - write_cmnd, ref_cmd, sg, idx, - page, page->net_priv); - sendpage = sock_no_sendpage; - } - } else - page->net_priv = ref_cmd; - spin_unlock(&net_priv_lock); - } -#endif sendsize = min(size, length); if (size <= sendsize) { retry2: @@ -1517,11 +1251,10 @@ retry2: goto out_res; } - check_net_priv(ref_cmd, page); if (res == size) { conn->write_size = 0; res = saved_size; - goto out_put; + goto out; } offset += res; @@ -1545,8 +1278,6 @@ retry1: goto out_res; } - check_net_priv(ref_cmd, page); - size -= res; if (res == sendsize) { @@ -1568,20 +1299,15 @@ out_off: out_iov: conn->write_size = size; if ((saved_size == size) && res == -EAGAIN) - goto out_put; + goto out; res = saved_size - size; -out_put: - if (do_put) - __iscsi_put_page_callback(ref_cmd); - out: TRACE_EXIT_RES(res); return res; out_res: - check_net_priv(ref_cmd, page); if (res == -EAGAIN) goto out_off; /* else go through */ @@ -1605,7 +1331,7 @@ out_err: scst_set_delivery_status(ref_cmd->scst_cmd, SCST_CMD_DELIVERY_FAILED); } - goto out_put; + goto out; } static int exit_tx(struct iscsi_conn *conn, int res)