scst/iscsi-scst/kernel/iscsi.c

/*
 *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
 *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin
 *  Copyright (C) 2007 - 2008 CMS Distribution Limited
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation, version 2
 *  of the License.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 */

#include <linux/module.h>
#include <linux/hash.h>
#include <linux/kthread.h>
#include <net/tcp.h>
#include <scsi/scsi.h>

#include "iscsi.h"
#include "digest.h"

#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
#warning "Patch put_page_callback-<kernel-version>.patch not applied on your\
 kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION\
 config option not set. ISCSI-SCST will be working with not the best\
 performance. Refer README file for details."
#endif

#define ISCSI_INIT_WRITE_WAKE		0x1
#define ISCSI_INIT_WRITE_REMOVE_HASH	0x2

static int ctr_major;
static char ctr_name[] = "iscsi-scst-ctl";
static int iscsi_template_registered;

#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
#endif

static struct kmem_cache *iscsi_cmnd_cache;

DEFINE_SPINLOCK(iscsi_rd_lock);
LIST_HEAD(iscsi_rd_list);
DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);

DEFINE_SPINLOCK(iscsi_wr_lock);
LIST_HEAD(iscsi_wr_list);
DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);

static struct page *dummy_page;
static struct scatterlist dummy_sg;

static uint8_t sense_fixed_unexpected_unsolicited_data[SCST_STANDARD_SENSE_LEN];
static uint8_t sense_descr_unexpected_unsolicited_data[SCST_STANDARD_SENSE_LEN];

struct iscsi_thread_t {
	struct task_struct *thr;
	struct list_head threads_list_entry;
};

static LIST_HEAD(iscsi_threads_list);

static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd);
static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
static void req_cmnd_release(struct iscsi_cmnd *req);

static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
{
	struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);

	if (hdr->flags & ISCSI_CMD_WRITE)
		return be32_to_cpu(hdr->data_length);
	return 0;
}

static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
{
	struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);

	if (hdr->flags & ISCSI_CMD_READ) {
		struct iscsi_ahs_hdr *ahdr;

		if (!(hdr->flags & ISCSI_CMD_WRITE))
			return be32_to_cpu(hdr->data_length);

		ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
		if (ahdr != NULL) {
			uint8_t *p = (uint8_t *)ahdr;
			int size = 0;
			do {
				int s;

				ahdr = (struct iscsi_ahs_hdr *)p;

				if (ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH) {
					struct iscsi_rlength_ahdr *rh =
					      (struct iscsi_rlength_ahdr *)ahdr;
					return be32_to_cpu(rh->read_length);
				}

				s = 3 + be16_to_cpu(ahdr->ahslength);
				s = (s + 3) & -4;
				size += s;
				p += s;
			} while (size < cmnd->pdu.ahssize);
		}
	}
	return 0;
}

static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
{
	EXTRACHECKS_BUG_ON(cmnd->data_waiting);

	cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
	scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
		SCST_CONTEXT_THREAD);
	return;
}

static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
{
	/*
	 * There is no race with conn_abort(), since all functions
	 * called from single read thread
	 */
	iscsi_extracheck_is_rd_thread(cmnd->conn);
	cmnd->data_waiting = 0;

	iscsi_restart_cmnd(cmnd);
	return;
}

static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
{
	TRACE_MGMT_DBG("Failing data waiting cmd %p", cmnd);

	/*
	 * There is no race with conn_abort(), since all functions
	 * called from single read thread
	 */
	iscsi_extracheck_is_rd_thread(cmnd->conn);
	cmnd->data_waiting = 0;

	req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
}

struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
			      struct iscsi_cmnd *parent)
{
	struct iscsi_cmnd *cmnd;

	/* ToDo: __GFP_NOFAIL?? */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
	cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
	memset(cmnd, 0, sizeof(*cmnd));
#else
	cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
#endif

	atomic_set(&cmnd->ref_cnt, 1);
	cmnd->scst_state = ISCSI_CMD_STATE_NEW;
	cmnd->conn = conn;
	cmnd->parent_req = parent;
	init_waitqueue_head(&cmnd->scst_waitQ);

	if (parent == NULL) {
		conn_get(conn);

#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
		atomic_set(&cmnd->net_ref_cnt, 0);
#endif
		spin_lock_init(&cmnd->rsp_cmd_lock);
		INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
		INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);

		spin_lock_bh(&conn->cmd_list_lock);
		list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
		spin_unlock_bh(&conn->cmd_list_lock);
	}

	TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
	return cmnd;
}

/* Frees a command. Also frees the additional header. */
static void cmnd_free(struct iscsi_cmnd *cmnd)
{
	TRACE_DBG("%p", cmnd);

	if (unlikely(cmnd->tm_aborted)) {
		TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
			"parent_req %p)", cmnd, cmnd->scst_cmd,
			cmnd->scst_state, cmnd->parent_req);
	}

	/* Catch users from cmd_list or rsp_cmd_list */
	EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);

	kfree(cmnd->pdu.ahs);

	if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
		struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);

		PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
			"%x, %x, %x, %x", cmnd, req->opcode, req->scb[0],
			req->flags, req->itt, be32_to_cpu(req->data_length),
			req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));

		if (unlikely(cmnd->parent_req)) {
			struct iscsi_scsi_cmd_hdr *preq =
					cmnd_hdr(cmnd->parent_req);
			PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode,
				preq->scb[0]);
		}
		sBUG();
	}

	kmem_cache_free(iscsi_cmnd_cache, cmnd);
	return;
}

void cmnd_done(struct iscsi_cmnd *cmnd)
{
	TRACE_DBG("%p", cmnd);

	if (unlikely(cmnd->tm_aborted)) {
		TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
			"parent_req %p)", cmnd, cmnd->scst_cmd,
			cmnd->scst_state, cmnd->parent_req);
	}

	EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);

	if (cmnd->on_written_list) {
		struct iscsi_conn *conn = cmnd->conn;
		TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
			conn);
		spin_lock_bh(&conn->write_list_lock);
		list_del(&cmnd->write_list_entry);
		cmnd->on_written_list = 0;
		spin_unlock_bh(&conn->write_list_lock);
	}

	if (cmnd->parent_req == NULL) {
		struct iscsi_conn *conn = cmnd->conn;
		TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);

		spin_lock_bh(&conn->cmd_list_lock);
		list_del(&cmnd->cmd_list_entry);
		spin_unlock_bh(&conn->cmd_list_lock);

		conn_put(conn);

		EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
		EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));

		/* Order between above and below code is important! */

		if ((cmnd->scst_cmd != NULL) || (cmnd->scst_aen != NULL)) {
			switch (cmnd->scst_state) {
			case ISCSI_CMD_STATE_PROCESSED:
				TRACE_DBG("cmd %p PROCESSED", cmnd);
				scst_tgt_cmd_done(cmnd->scst_cmd,
					SCST_CONTEXT_DIRECT);
				break;

			case ISCSI_CMD_STATE_AFTER_PREPROC:
			{
				struct scst_cmd *scst_cmd = cmnd->scst_cmd;
				TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
				cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
				cmnd->scst_cmd = NULL;
				scst_restart_cmd(scst_cmd,
					SCST_PREPROCESS_STATUS_ERROR_FATAL,
					SCST_CONTEXT_THREAD);
				break;
			}

			case ISCSI_CMD_STATE_AEN:
				TRACE_DBG("cmd %p AEN PROCESSED", cmnd);
				scst_aen_done(cmnd->scst_aen);
				break;

			default:
				PRINT_CRIT_ERROR("Unexpected cmnd scst state "
					"%d", cmnd->scst_state);
				sBUG();
				break;
			}
		}
	} else {
		TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
			cmnd->parent_req);

		spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
		list_del(&cmnd->rsp_cmd_list_entry);
		spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);

		cmnd_put(cmnd->parent_req);
	}

	/* Order between above and below code is important! */

	if (cmnd->own_sg) {
		TRACE_DBG("%s", "own_sg");
		if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
			scst_free(cmnd->sg, cmnd->sg_cnt);
#ifdef CONFIG_SCST_DEBUG
		cmnd->own_sg = 0;
		cmnd->sg = NULL;
		cmnd->sg_cnt = -1;
#endif
	}

	if (cmnd->dec_active_cmnds) {
		struct iscsi_session *sess = cmnd->conn->session;
		TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
			"new value %d)", cmnd, sess,
			atomic_read(&sess->active_cmds)-1);
		atomic_dec(&sess->active_cmds);
#ifdef CONFIG_SCST_EXTRACHECKS
		if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
			PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
				atomic_read(&sess->active_cmds));
			sBUG();
		}
#endif
	}

	cmnd_free(cmnd);
	return;
}

/*
 * Corresponding conn may also gets destroyed atfer this function, except only
 * if it's called from the read thread!
 *
 * It can't be called in parallel with iscsi_cmnds_init_write()!
 */
void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
{
	struct iscsi_cmnd *rsp, *t;
	struct iscsi_conn *conn = req->conn;
	LIST_HEAD(cmds_list);

	TRACE_ENTRY();

	TRACE_MGMT_DBG("%p", req);

	sBUG_ON(req == conn->read_cmnd);

	if (flags & ISCSI_FORCE_RELEASE_WRITE) {
		spin_lock_bh(&conn->write_list_lock);
		list_for_each_entry_safe(rsp, t, &conn->write_list,
						write_list_entry) {
			if (rsp->parent_req != req)
				continue;

			cmd_del_from_write_list(rsp);

			list_add_tail(&rsp->write_list_entry, &cmds_list);
		}
		spin_unlock_bh(&conn->write_list_lock);

		list_for_each_entry_safe(rsp, t, &cmds_list,
						write_list_entry) {
			TRACE_MGMT_DBG("Putting write rsp %p", rsp);
			list_del(&rsp->write_list_entry);
			cmnd_put(rsp);
		}
	}

again_rsp:
	spin_lock_bh(&req->rsp_cmd_lock);
	list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
			rsp_cmd_list_entry) {
		bool r;

		if (rsp->force_cleanup_done)
			continue;

		rsp->force_cleanup_done = 1;

		if (cmnd_get_check(rsp))
			continue;

		spin_unlock_bh(&req->rsp_cmd_lock);

		spin_lock_bh(&conn->write_list_lock);
		r = rsp->on_write_list || rsp->write_processing_started;
		spin_unlock_bh(&conn->write_list_lock);

		cmnd_put(rsp);

		if (r)
			goto again_rsp;

		/*
		 * If both on_write_list and write_processing_started not set,
		 * we can safely put() rsp.
		 */
		TRACE_MGMT_DBG("Putting rsp %p", rsp);
		cmnd_put(rsp);
		goto again_rsp;
	}
	spin_unlock_bh(&req->rsp_cmd_lock);

	req_cmnd_release(req);

	TRACE_EXIT();
	return;
}

/*
 * Corresponding conn may also gets destroyed atfer this function, except only
 * if it's called from the read thread!
 */
static void req_cmnd_release(struct iscsi_cmnd *req)
{
	struct iscsi_cmnd *c, *t;

	TRACE_ENTRY();

	TRACE_DBG("%p", req);

#ifdef CONFIG_SCST_EXTRACHECKS
	sBUG_ON(req->release_called);
	req->release_called = 1;
#endif

	if (unlikely(req->tm_aborted)) {
		TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
			"state %d)", req, req->scst_cmd, req->scst_state);
	}

	sBUG_ON(req->parent_req != NULL);

	list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
				rx_ddigest_cmd_list_entry) {
		cmd_del_from_rx_ddigest_list(c);
		cmnd_put(c);
	}

	if (req->hashed)
		cmnd_remove_hash(req);

	if (req->dec_active_cmnds) {
		struct iscsi_session *sess = req->conn->session;
		TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
			"new value %d)", req, sess,
			atomic_read(&sess->active_cmds)-1);
		atomic_dec(&sess->active_cmds);
		req->dec_active_cmnds = 0;
#ifdef CONFIG_SCST_EXTRACHECKS
		if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
			PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
				atomic_read(&sess->active_cmds));
			sBUG();
		}
#endif
	}

	cmnd_put(req);

	TRACE_EXIT();
	return;
}

/*
 * Corresponding conn may also gets destroyed atfer this function, except only
 * if it's called from the read thread!
 */
void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
{
	TRACE_DBG("%p", cmnd);

#ifdef CONFIG_SCST_EXTRACHECKS
	sBUG_ON(cmnd->release_called);
	cmnd->release_called = 1;
#endif

	sBUG_ON(cmnd->hashed);
	sBUG_ON(cmnd->parent_req == NULL);

	cmnd_put(cmnd);
	return;
}

/**
 * create a new command used as response.
 *
 * iscsi_cmnd_create_rsp_cmnd -
 * @cmnd: ptr to request command
 *
 * @return    ptr to response command or NULL
 */
static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
{
	struct iscsi_cmnd *rsp;

	rsp = cmnd_alloc(parent->conn, parent);

	spin_lock_bh(&parent->rsp_cmd_lock);
	TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
	list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
	spin_unlock_bh(&parent->rsp_cmd_lock);
	cmnd_get(parent);
	return rsp;
}

static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
{
	struct iscsi_cmnd *res = NULL;

	/* Currently this lock isn't needed, but just in case.. */
	spin_lock_bh(&req->rsp_cmd_lock);
	if (!list_empty(&req->rsp_cmd_list)) {
		res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
			rsp_cmd_list_entry);
	}
	spin_unlock_bh(&req->rsp_cmd_lock);

	return res;
}

static void iscsi_cmnds_init_write(struct list_head *send, int flags)
{
	struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
						write_list_entry);
	struct iscsi_conn *conn = rsp->conn;
	struct list_head *pos, *next;

	sBUG_ON(list_empty(send));

	/*
	 * If we don't remove hashed req cmd from the hash list here, before
	 * submitting it for transmittion, we will have a race, when for
	 * some reason cmd's release is delayed after transmittion and
	 * initiator sends cmd with the same ITT => this command will be
	 * erroneously rejected as a duplicate.
	 */
	if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) &&
	    rsp->parent_req->hashed &&
	    (rsp->parent_req->r2t_length == 0) &&
	    (rsp->parent_req->outstanding_r2t == 0))
		cmnd_remove_hash(rsp->parent_req);

	if (!(conn->ddigest_type & DIGEST_NONE)) {
		list_for_each(pos, send) {
			rsp = list_entry(pos, struct iscsi_cmnd,
						write_list_entry);

			if (rsp->pdu.datasize != 0) {
				TRACE_DBG("Doing data digest (%p:%x)", rsp,
					cmnd_opcode(rsp));
				digest_tx_data(rsp);
			}
		}
	}

	spin_lock_bh(&conn->write_list_lock);
	list_for_each_safe(pos, next, send) {
		rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);

		TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));

		sBUG_ON(conn != rsp->conn);

		list_del(&rsp->write_list_entry);
		cmd_add_on_write_list(conn, rsp);
	}
	spin_unlock_bh(&conn->write_list_lock);

	if (flags & ISCSI_INIT_WRITE_WAKE)
		iscsi_make_conn_wr_active(conn);

	return;
}

static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
{
	LIST_HEAD(head);

	if (unlikely(rsp->on_write_list)) {
		PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %x %u "
			"%u %u %u %u %u %u %d %d",
			cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
			cmnd_scsicode(rsp), rsp->r2t_sn,
			rsp->r2t_length, rsp->is_unsolicited_data,
			rsp->target_task_tag, rsp->outstanding_r2t,
			rsp->hdigest, rsp->ddigest,
			list_empty(&rsp->rsp_cmd_list), rsp->hashed);
		sBUG();
	}
	list_add_tail(&rsp->write_list_entry, &head);
	iscsi_cmnds_init_write(&head, flags);
	return;
}

static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
{
	struct iscsi_cmnd *rsp;
	struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
	struct iscsi_data_in_hdr *rsp_hdr;
	u32 pdusize, expsize, scsisize, size, offset, sn;
	LIST_HEAD(send);

	TRACE_DBG("req %p", req);

	pdusize = req->conn->session->sess_param.max_xmit_data_length;
	expsize = cmnd_read_size(req);
	size = min(expsize, (u32)req->bufflen);
	offset = 0;
	sn = 0;

	while (1) {
		rsp = iscsi_cmnd_create_rsp_cmnd(req);
		TRACE_DBG("rsp %p", rsp);
		rsp->sg = req->sg;
		rsp->sg_cnt = req->sg_cnt;
		rsp->bufflen = req->bufflen;
		rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;

		rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
		rsp_hdr->itt = req_hdr->itt;
		rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
		rsp_hdr->buffer_offset = cpu_to_be32(offset);
		rsp_hdr->data_sn = cpu_to_be32(sn);

		if (size <= pdusize) {
			TRACE_DBG("offset %d, size %d", offset, size);
			rsp->pdu.datasize = size;
			if (send_status) {
				TRACE_DBG("status %x", status);
				rsp_hdr->flags =
					ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
				rsp_hdr->cmd_status = status;
			}
			scsisize = req->bufflen;
			if (scsisize < expsize) {
				rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
				size = expsize - scsisize;
			} else if (scsisize > expsize) {
				rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
				size = scsisize - expsize;
			} else
				size = 0;
			rsp_hdr->residual_count = cpu_to_be32(size);
			list_add_tail(&rsp->write_list_entry, &send);
			break;
		}

		TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
			size);

		rsp->pdu.datasize = pdusize;

		size -= pdusize;
		offset += pdusize;
		sn++;

		list_add_tail(&rsp->write_list_entry, &send);
	}
	iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
	return;
}

static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
	const u8 *sense_buf, int sense_len)
{
	struct iscsi_cmnd *rsp;
	struct iscsi_scsi_rsp_hdr *rsp_hdr;
	struct scatterlist *sg;

	rsp = iscsi_cmnd_create_rsp_cmnd(req);
	TRACE_DBG("%p", rsp);

	rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
	rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
	rsp_hdr->flags = ISCSI_FLG_FINAL;
	rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
	rsp_hdr->cmd_status = status;
	rsp_hdr->itt = cmnd_hdr(req)->itt;

	if (SCST_SENSE_VALID(sense_buf)) {
		TRACE_DBG("%s", "SENSE VALID");

		sg = rsp->sg = rsp->rsp_sg;
		rsp->sg_cnt = 2;
		rsp->own_sg = 1;

		sg_init_table(sg, 2);
		sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
		sg_set_buf(&sg[1], sense_buf, sense_len);

		rsp->sense_hdr.length = cpu_to_be16(sense_len);

		rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
		rsp->bufflen = rsp->pdu.datasize;
	} else {
		rsp->pdu.datasize = 0;
		rsp->bufflen = 0;
	}

	return rsp;
}

static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
{
	struct iscsi_cmnd *rsp;
	struct iscsi_reject_hdr *rsp_hdr;
	struct scatterlist *sg;

	TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);

	sBUG_ON(req->rejected);
	req->rejected = 1;
	req->reject_reason = ISCSI_REJECT_CMD;

	rsp = iscsi_cmnd_create_rsp_cmnd(req);
	rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;

	rsp_hdr->opcode = ISCSI_OP_REJECT;
	rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
	rsp_hdr->reason = reason;

	sg = rsp->sg = rsp->rsp_sg;
	rsp->sg_cnt = 1;
	rsp->own_sg = 1;
	sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
	rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);

	iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
					 ISCSI_INIT_WRITE_WAKE);

	cmnd_prepare_get_rejected_cmd_data(req);
	return;
}

static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
{
	int res = max(-1, (int)sess->max_queued_cmnds -
				atomic_read(&sess->active_cmds)-1);
	TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
		sess, atomic_read(&sess->active_cmds));
	return res;
}

static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
{
	struct iscsi_conn *conn = cmnd->conn;
	struct iscsi_session *sess = conn->session;
	u32 res;

	spin_lock(&sess->sn_lock);

	if (set_stat_sn)
		cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
	cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
	cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
				 iscsi_get_allowed_cmds(sess));

	res = cpu_to_be32(conn->stat_sn);

	spin_unlock(&sess->sn_lock);
	return res;
}

/* Called under sn_lock */
static void __update_stat_sn(struct iscsi_cmnd *cmnd)
{
	struct iscsi_conn *conn = cmnd->conn;
	u32 exp_stat_sn;

	cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
	TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
	if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
	    (int)(exp_stat_sn - conn->stat_sn) <= 0) {
		/* free pdu resources */
		cmnd->conn->exp_stat_sn = exp_stat_sn;
	}
	return;
}

static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
{
	spin_lock(&cmnd->conn->session->sn_lock);
	__update_stat_sn(cmnd);
	spin_unlock(&cmnd->conn->session->sn_lock);
	return;
}

/* Called under sn_lock */
static int check_cmd_sn(struct iscsi_cmnd *cmnd)
{
	struct iscsi_session *session = cmnd->conn->session;
	u32 cmd_sn;

	cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
	TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
	if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
		return 0;
	PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
	return -ISCSI_REASON_PROTOCOL_ERROR;
}

static inline struct iscsi_cmnd *__cmnd_find_hash(
	struct iscsi_session *session, u32 itt, u32 ttt)
{
	struct list_head *head;
	struct iscsi_cmnd *cmnd;

	head = &session->cmnd_hash[cmnd_hashfn(itt)];

	list_for_each_entry(cmnd, head, hash_list_entry) {
		if (cmnd->pdu.bhs.itt == itt) {
			if (ttt != ISCSI_RESERVED_TAG &&
			    ttt != cmnd->target_task_tag)
				continue;
			return cmnd;
		}
	}
	return NULL;
}

static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
	u32 itt, u32 ttt)
{
	struct iscsi_cmnd *cmnd;

	spin_lock(&session->cmnd_hash_lock);
	cmnd = __cmnd_find_hash(session, itt, ttt);
	spin_unlock(&session->cmnd_hash_lock);

	return cmnd;
}

static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
	u32 itt, u32 ttt)
{
	struct iscsi_cmnd *cmnd;

	spin_lock(&session->cmnd_hash_lock);
	cmnd = __cmnd_find_hash(session, itt, ttt);
	if (cmnd != NULL) {
		if (unlikely(cmnd_get_check(cmnd)))
			cmnd = NULL;
	}
	spin_unlock(&session->cmnd_hash_lock);

	return cmnd;
}

static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
{
	struct iscsi_session *session = cmnd->conn->session;
	struct iscsi_cmnd *tmp;
	struct list_head *head;
	int err = 0;
	u32 itt = cmnd->pdu.bhs.itt;

	TRACE_DBG("%p:%x", cmnd, itt);
	if (unlikely(itt == ISCSI_RESERVED_TAG)) {
		PRINT_ERROR("%s", "ITT is RESERVED_TAG");
		PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
			sizeof(cmnd->pdu.bhs));
		err = -ISCSI_REASON_PROTOCOL_ERROR;
		goto out;
	}

	spin_lock(&session->cmnd_hash_lock);

	head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];

	tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
	if (likely(!tmp)) {
		list_add_tail(&cmnd->hash_list_entry, head);
		cmnd->hashed = 1;
	} else {
		PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
		err = -ISCSI_REASON_TASK_IN_PROGRESS;
	}

	spin_unlock(&session->cmnd_hash_lock);

	if (likely(!err)) {
		spin_lock(&session->sn_lock);
		__update_stat_sn(cmnd);
		err = check_cmd_sn(cmnd);
		spin_unlock(&session->sn_lock);
	}

out:
	return err;
}

static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
{
	struct iscsi_session *session = cmnd->conn->session;
	struct iscsi_cmnd *tmp;

	spin_lock(&session->cmnd_hash_lock);

	tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);

	if (likely(tmp && tmp == cmnd)) {
		list_del(&cmnd->hash_list_entry);
		cmnd->hashed = 0;
	} else {
		PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
	}

	spin_unlock(&session->cmnd_hash_lock);
	return;
}

static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
{
	struct iscsi_conn *conn = cmnd->conn;
	struct scatterlist *sg = cmnd->sg;
	char __user *addr;
	u32 size;
	unsigned int i;

	TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
		cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
		cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);

	iscsi_extracheck_is_rd_thread(conn);

	size = cmnd->pdu.datasize;
	if (!size)
		return;

	if (sg == NULL) {
		/*
		 * There are no problems with the safety from concurrent
		 * accesses to dummy_page in dummy_sg, since data only
		 * will be read and then discarded.
		 */
		sg = cmnd->sg = &dummy_sg;
		cmnd->bufflen = PAGE_SIZE;
		cmnd->own_sg = 1;
	}

	addr = (char __force __user *)(page_address(sg_page(&sg[0])));
	sBUG_ON(addr == NULL);
	conn->read_size = size;
	for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
		/* We already checked pdu.datasize in check_segment_length() */
		sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
		conn->read_iov[i].iov_base = addr;
		conn->read_iov[i].iov_len = cmnd->bufflen;
	}
	conn->read_iov[i].iov_base = addr;
	conn->read_iov[i].iov_len = size;
	conn->read_msg.msg_iov = conn->read_iov;
	conn->read_msg.msg_iovlen = ++i;

	return;
}

static void cmnd_reject_scsi_cmd(struct iscsi_cmnd *req)
{
	struct iscsi_cmnd *rsp;
	struct iscsi_scsi_rsp_hdr *rsp_hdr;
	u32 size;

	TRACE_DBG("%p", req);

	sBUG_ON(req->rejected);
	req->rejected = 1;
	req->reject_reason = ISCSI_REJECT_SCSI_CMD;

	rsp = get_rsp_cmnd(req);
	if (rsp == NULL) {
		/* That can be true for aborted commands */
		goto out_reject;
	}

	rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;

	sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);

	size = cmnd_write_size(req);
	if (size) {
		rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
		rsp_hdr->residual_count = cpu_to_be32(size);
	}
	size = cmnd_read_size(req);
	if (size) {
		if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
			rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
			rsp_hdr->bi_residual_count = cpu_to_be32(size);
		} else {
			rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
			rsp_hdr->residual_count = cpu_to_be32(size);
		}
	}

	iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
					 ISCSI_INIT_WRITE_WAKE);

out_reject:
	cmnd_prepare_get_rejected_cmd_data(req);
	return;
}

static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
	struct iscsi_cmnd *cmd,	u32 offset, u32 size)
{
	struct scatterlist *sg = cmd->sg;
	unsigned int bufflen = cmd->bufflen;
	unsigned int idx, i;
	char __user *addr;
	int res = 0;

	TRACE_DBG("%p %u,%u", cmd->sg, offset, size);

	iscsi_extracheck_is_rd_thread(conn);

	if (unlikely((offset >= bufflen) ||
		     (offset + size > bufflen))) {
		PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
		mark_conn_closed(conn);
		res = -EIO;
		goto out;
	}

	offset += sg[0].offset;
	idx = offset >> PAGE_SHIFT;
	offset &= ~PAGE_MASK;

	conn->read_msg.msg_iov = conn->read_iov;
	conn->read_size = size;

	i = 0;
	while (1) {
		addr = (char __force __user *)(page_address(sg_page(&sg[idx])));
		sBUG_ON(addr == NULL);
		conn->read_iov[i].iov_base = addr + offset;
		if (offset + size <= PAGE_SIZE) {
			TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
				idx, offset, size, addr);
			conn->read_iov[i].iov_len = size;
			conn->read_msg.msg_iovlen = ++i;
			break;
		}
		conn->read_iov[i].iov_len = PAGE_SIZE - offset;
		TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
			idx, offset, size, conn->read_iov[i].iov_len, addr);
		size -= conn->read_iov[i].iov_len;
		if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
			PRINT_ERROR("Initiator %s violated negotiated "
				"parameters by sending too much data (size "
				"left %d)", conn->session->initiator_name,
				size);
			mark_conn_closed(conn);
			res = -EINVAL;
			break;
		}
		idx++;
		offset = sg[idx].offset;
	}
	TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
		conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);

out:
	return res;
}

static void send_r2t(struct iscsi_cmnd *req)
{
	struct iscsi_session *session = req->conn->session;
	struct iscsi_cmnd *rsp;
	struct iscsi_r2t_hdr *rsp_hdr;
	u32 offset, burst;
	LIST_HEAD(send);

	if (unlikely(req->tm_aborted)) {
		TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
			"(r2t_length %d, outstanding_r2t %d)", req,
			req->scst_cmd, req->r2t_length, req->outstanding_r2t);
		if (req->outstanding_r2t == 0)
			iscsi_fail_waiting_cmnd(req);
		goto out;
	}

	/*
	 * There is no race with data_out_start() and conn_abort(), since
	 * all functions called from single read thread
	 */
	iscsi_extracheck_is_rd_thread(req->conn);

	burst = session->sess_param.max_burst_length;
	offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;

	do {
		rsp = iscsi_cmnd_create_rsp_cmnd(req);
		rsp->pdu.bhs.ttt = req->target_task_tag;
		rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
		rsp_hdr->opcode = ISCSI_OP_R2T;
		rsp_hdr->flags = ISCSI_FLG_FINAL;
		rsp_hdr->lun = cmnd_hdr(req)->lun;
		rsp_hdr->itt = cmnd_hdr(req)->itt;
		rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
		rsp_hdr->buffer_offset = cpu_to_be32(offset);
		if (req->r2t_length > burst) {
			rsp_hdr->data_length = cpu_to_be32(burst);
			req->r2t_length -= burst;
			offset += burst;
		} else {
			rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
			req->r2t_length = 0;
		}

		TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
			be32_to_cpu(rsp_hdr->data_length),
			be32_to_cpu(rsp_hdr->buffer_offset),
			be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);

		list_add_tail(&rsp->write_list_entry, &send);

		if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
			break;

	} while (req->r2t_length != 0);

	iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);

out:
	return;
}

static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
{
	int res = SCST_PREPROCESS_STATUS_SUCCESS;
	struct iscsi_cmnd *req = (struct iscsi_cmnd *)
		scst_cmd_get_tgt_priv(scst_cmd);
	struct iscsi_cmnd *c, *t;

	TRACE_ENTRY();

	EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));

	if (scst_cmd_get_data_direction(scst_cmd) == SCST_DATA_READ) {
		EXTRACHECKS_BUG_ON(!list_empty(&req->rx_ddigest_cmd_list));
		goto out;
	}

	/* If data digest isn't used this list will be empty */
	list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
				rx_ddigest_cmd_list_entry) {
		TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
		if (digest_rx_data(c) != 0) {
			scst_set_cmd_error(scst_cmd,
				SCST_LOAD_SENSE(iscsi_sense_crc_error));
			res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
			/*
			 * The rest of rx_ddigest_cmd_list will be freed
			 * in req_cmnd_release()
			 */
			goto out;
		}
		cmd_del_from_rx_ddigest_list(c);
		cmnd_put(c);
	}

out:
	TRACE_EXIT_RES(res);
	return res;
}

static int noop_out_start(struct iscsi_cmnd *cmnd)
{
	struct iscsi_conn *conn = cmnd->conn;
	u32 size, tmp;
	int i, err = 0;

	TRACE_DBG("%p", cmnd);

	iscsi_extracheck_is_rd_thread(conn);

	if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
		/*
		 * We don't request a NOP-Out by sending a NOP-In.
		 * See 10.18.2 in the draft 20.
		 */
		PRINT_ERROR("Initiator sent command with not RESERVED tag and "
			"TTT %x", cmnd_itt(cmnd));
		err = -ISCSI_REASON_PROTOCOL_ERROR;
		goto out;
	}

	if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
		if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
			PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
				"non-immediate command");
		spin_lock(&conn->session->sn_lock);
		__update_stat_sn(cmnd);
		err = check_cmd_sn(cmnd);
		spin_unlock(&conn->session->sn_lock);
		if (unlikely(err))
			goto out;
	} else {
		err = cmnd_insert_hash(cmnd);
		if (unlikely(err < 0)) {
			PRINT_ERROR("Can't insert in hash: ignore this "
				"request %x", cmnd_itt(cmnd));
			goto out;
		}
	}

	size = cmnd->pdu.datasize;

	if (size) {
		conn->read_msg.msg_iov = conn->read_iov;
		if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
			struct scatterlist *sg;

			cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
						&cmnd->sg_cnt);
			if (sg == NULL) {
				TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for"
				      " %d NOP-Out payload failed", size);
				err = -ISCSI_REASON_OUT_OF_RESOURCES;
				goto out;
			}

			/* We already checked it in check_segment_length() */
			sBUG_ON(cmnd->sg_cnt > (signed)ISCSI_CONN_IOV_MAX);

			cmnd->own_sg = 1;
			cmnd->bufflen = size;

			for (i = 0; i < cmnd->sg_cnt; i++) {
				conn->read_iov[i].iov_base =
					(void __force __user *)(page_address(sg_page(&sg[i])));
				tmp = min_t(u32, size, PAGE_SIZE);
				conn->read_iov[i].iov_len = tmp;
				conn->read_size += tmp;
				size -= tmp;
			}
			sBUG_ON(size != 0);
		} else {
			/*
			 * There are no problems with the safety from concurrent
			 * accesses to dummy_page, since for ISCSI_RESERVED_TAG
			 * the data only read and then discarded.
			 */
			for (i = 0; i < (signed)ISCSI_CONN_IOV_MAX; i++) {
				conn->read_iov[i].iov_base =
					(void __force __user *)(page_address(dummy_page));
				tmp = min_t(u32, size, PAGE_SIZE);
				conn->read_iov[i].iov_len = tmp;
				conn->read_size += tmp;
				size -= tmp;
			}

			/* We already checked size in check_segment_length() */
			sBUG_ON(size != 0);
		}

		conn->read_msg.msg_iovlen = i;
		TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
			conn->read_msg.msg_iovlen);
	}

out:
	return err;
}

static inline u32 get_next_ttt(struct iscsi_conn *conn)
{
	u32 ttt;
	struct iscsi_session *session = conn->session;

	iscsi_extracheck_is_rd_thread(conn);

	if (session->next_ttt == ISCSI_RESERVED_TAG)
		session->next_ttt++;
	ttt = session->next_ttt++;

	return cpu_to_be32(ttt);
}

static int scsi_cmnd_start(struct iscsi_cmnd *req)
{
	struct iscsi_conn *conn = req->conn;
	struct iscsi_session *session = conn->session;
	struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
	struct scst_cmd *scst_cmd;
	scst_data_direction dir;
	struct iscsi_ahs_hdr *ahdr;
	int res = 0;

	TRACE_ENTRY();

	TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);

	TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
		"new value %d)", req, session,
		atomic_read(&session->active_cmds)+1);
	atomic_inc(&session->active_cmds);
	req->dec_active_cmnds = 1;

	scst_cmd = scst_rx_cmd(session->scst_sess,
		(uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
		req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
	if (scst_cmd == NULL) {
		create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
		cmnd_reject_scsi_cmd(req);
		goto out;
	}

	req->scst_cmd = scst_cmd;
	scst_cmd_set_tag(scst_cmd, req_hdr->itt);
	scst_cmd_set_tgt_priv(scst_cmd, req);

	if (req_hdr->flags & ISCSI_CMD_READ) {
		dir = SCST_DATA_READ;
#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
		scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
#endif
	} else if (req_hdr->flags & ISCSI_CMD_WRITE)
		dir = SCST_DATA_WRITE;
	else
		dir = SCST_DATA_NONE;
	scst_cmd_set_expected(scst_cmd, dir,
		be32_to_cpu(req_hdr->data_length));

	switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
	case ISCSI_CMD_SIMPLE:
		scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
		break;
	case ISCSI_CMD_HEAD_OF_QUEUE:
		scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
		break;
	case ISCSI_CMD_ORDERED:
		scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
		break;
	case ISCSI_CMD_ACA:
		scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
		break;
	case ISCSI_CMD_UNTAGGED:
		scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
		break;
	default:
		PRINT_ERROR("Unknown task code %x, use ORDERED instead",
			req_hdr->flags & ISCSI_CMD_ATTR_MASK);
		scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
		break;
	}

	/* cmd_sn is already in CPU format converted in check_cmd_sn() */
	scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);

	ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
	if (ahdr != NULL) {
		uint8_t *p = (uint8_t *)ahdr;
		int size = 0;
		do {
			int s;

			ahdr = (struct iscsi_ahs_hdr *)p;

			if (ahdr->ahstype == ISCSI_AHSTYPE_CDB) {
				struct iscsi_cdb_ahdr *eca =
					(struct iscsi_cdb_ahdr *)ahdr;
				scst_cmd_set_ext_cdb(scst_cmd, eca->cdb,
					be16_to_cpu(ahdr->ahslength) - 1);
				break;
			}
			s = 3 + be16_to_cpu(ahdr->ahslength);
			s = (s + 3) & -4;
			size += s;
			p += s;
		} while (size < req->pdu.ahssize);
	}

	TRACE_DBG("START Command (tag %d, queue_type %d)",
		req_hdr->itt, scst_cmd->queue_type);
	req->scst_state = ISCSI_CMD_STATE_RX_CMD;
	scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);

	wait_event(req->scst_waitQ, req->scst_state != ISCSI_CMD_STATE_RX_CMD);

	if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
		TRACE_DBG("req %p is in %x state", req, req->scst_state);
		if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
			cmnd_reject_scsi_cmd(req);
			goto out;
		}
		if (unlikely(req->tm_aborted)) {
			TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
				req->scst_cmd);
			cmnd_prepare_get_rejected_cmd_data(req);
			goto out;
		}
		sBUG();
	}

	dir = scst_cmd_get_data_direction(scst_cmd);
	if (dir != SCST_DATA_WRITE) {
		if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
			     req->pdu.datasize)) {
			PRINT_ERROR("Unexpected unsolicited data (ITT %x "
				"CDB %x", cmnd_itt(req), req_hdr->scb[0]);
			if (scst_get_cmd_dev_d_sense(scst_cmd))
				create_status_rsp(req, SAM_STAT_CHECK_CONDITION,
					sense_descr_unexpected_unsolicited_data,
					sizeof(sense_descr_unexpected_unsolicited_data));
			else
				create_status_rsp(req, SAM_STAT_CHECK_CONDITION,
					sense_fixed_unexpected_unsolicited_data,
					sizeof(sense_fixed_unexpected_unsolicited_data));
			cmnd_reject_scsi_cmd(req);
			goto out;
		}
	}

	if (dir == SCST_DATA_WRITE) {
		req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
		req->r2t_length = be32_to_cpu(req_hdr->data_length) -
					req->pdu.datasize;
		if (req->r2t_length > 0)
			req->data_waiting = 1;
	}
	req->target_task_tag = get_next_ttt(conn);
	req->sg = scst_cmd_get_sg(scst_cmd);
	req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
	req->bufflen = scst_cmd_get_bufflen(scst_cmd);
	if (unlikely(req->r2t_length > req->bufflen)) {
		PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
			req->r2t_length, req->bufflen);
		req->r2t_length = req->bufflen;
	}

	TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
		"r2t_length=%d, bufflen=%d", req, dir,
		req->is_unsolicited_data, req->r2t_length, req->bufflen);

	if (unlikely(!session->sess_param.immediate_data &&
		     req->pdu.datasize)) {
		PRINT_ERROR("Initiator %s violated negotiated paremeters: "
			"forbidden immediate data sent (ITT %x, op  %x)",
			session->initiator_name, cmnd_itt(req),
			req_hdr->scb[0]);
		res = -EINVAL;
		goto out;
	}

	if (unlikely(session->sess_param.initial_r2t &&
		     !(req_hdr->flags & ISCSI_CMD_FINAL))) {
		PRINT_ERROR("Initiator %s violated negotiated paremeters: "
			"initial R2T is required (ITT %x, op  %x)",
			session->initiator_name, cmnd_itt(req),
			req_hdr->scb[0]);
		res = -EINVAL;
		goto out;
	}

	if (req->pdu.datasize) {
		if (unlikely(dir != SCST_DATA_WRITE)) {
			PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't "
				"WRITE", req->pdu.datasize, dir);
			if (scst_get_cmd_dev_d_sense(scst_cmd))
				create_status_rsp(req, SAM_STAT_CHECK_CONDITION,
					sense_descr_unexpected_unsolicited_data,
					sizeof(sense_descr_unexpected_unsolicited_data));
			else
				create_status_rsp(req, SAM_STAT_CHECK_CONDITION,
					sense_fixed_unexpected_unsolicited_data,
					sizeof(sense_fixed_unexpected_unsolicited_data));
			cmnd_reject_scsi_cmd(req);
		} else
			res = cmnd_prepare_recv_pdu(conn, req, 0,
				req->pdu.datasize);
	}
out:
	/* Aborted commands will be freed in cmnd_rx_end() */
	TRACE_EXIT_RES(res);
	return res;
}

static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
{
	struct iscsi_data_out_hdr *req_hdr =
		(struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
	struct iscsi_cmnd *orig_req = NULL;
	u32 offset = be32_to_cpu(req_hdr->buffer_offset);
	int res = 0;

	TRACE_ENTRY();

	/*
	 * There is no race with send_r2t() and conn_abort(), since
	 * all functions called from single read thread
	 */
	iscsi_extracheck_is_rd_thread(cmnd->conn);

	update_stat_sn(cmnd);

	cmnd->cmd_req = orig_req = cmnd_find_hash(conn->session, req_hdr->itt,
					req_hdr->ttt);
	if (unlikely(orig_req == NULL)) {
		/* It might happen if req was aborted and then freed */
		TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
			cmnd_itt(cmnd), cmnd_ttt(cmnd));
		goto out_reject;
	}

	if (orig_req->is_unsolicited_data) {
		if (unlikely(orig_req->r2t_length < cmnd->pdu.datasize)) {
			PRINT_ERROR("Data size (%d) > R2T length (%d)",
				cmnd->pdu.datasize, orig_req->r2t_length);
			mark_conn_closed(conn);
			res = -EINVAL;
			goto out;
		}
		orig_req->r2t_length -= cmnd->pdu.datasize;
	}

	/* Check unsolicited burst data */
	if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
		     (orig_req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
		PRINT_ERROR("Unexpected data from %x %x",
			cmnd_itt(cmnd), cmnd_ttt(cmnd));
		mark_conn_closed(conn);
		res = -EINVAL;
		goto out;
	}

	TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, orig_req,
		offset, cmnd->pdu.datasize);

	res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);

out:
	TRACE_EXIT_RES(res);
	return res;

out_reject:
	sBUG_ON(cmnd->rejected);
	cmnd->rejected = 1;
	cmnd->reject_reason = ISCSI_REJECT_DATA;
	cmnd_prepare_get_rejected_cmd_data(cmnd);
	goto out;
}

static void data_out_end(struct iscsi_cmnd *cmnd)
{
	struct iscsi_data_out_hdr *req_hdr =
		(struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
	struct iscsi_cmnd *req;

	sBUG_ON(cmnd == NULL);
	req = cmnd->cmd_req;
	sBUG_ON(req == NULL);

	TRACE_DBG("cmnd %p, req %p", cmnd, req);

	iscsi_extracheck_is_rd_thread(cmnd->conn);

	if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
	    !cmnd->ddigest_checked) {
		cmd_add_on_rx_ddigest_list(req, cmnd);
		cmnd_get(cmnd);
	}

	if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
		TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
			req_hdr->flags & ISCSI_FLG_FINAL);

		if (req_hdr->flags & ISCSI_FLG_FINAL) {
			req->is_unsolicited_data = 0;
			if (req->pending)
				goto out_put;
		} else
			goto out_put;
	} else {
		TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
			req_hdr->flags & ISCSI_FLG_FINAL,
			req->outstanding_r2t, req->r2t_length);

		if (req_hdr->flags & ISCSI_FLG_FINAL) {
			if (unlikely(req->is_unsolicited_data)) {
				PRINT_ERROR("Unexpected unsolicited data "
					"(r2t_length %u, outstanding_r2t %d)",
					req->r2t_length,
					req->is_unsolicited_data);
				mark_conn_closed(req->conn);
				goto out_put;
			}
			req->outstanding_r2t--;
		} else
			goto out_put;
	}

	if (req->r2t_length != 0) {
		if (!req->is_unsolicited_data)
			send_r2t(req);
	} else
		iscsi_restart_waiting_cmnd(req);

out_put:
	cmnd_put(cmnd);
	return;
}

static void __cmnd_abort(struct iscsi_cmnd *cmnd)
{
	/*
	 * Here, if cmnd is data_waiting, we should iscsi_fail_waiting_cmnd()
	 * it. But, since this function can be called from any thread, not only
	 * from the read one, we at the moment can't do that, because of
	 * absence of appropriate locking protection. But this isn't a stuff
	 * for 1.0.0. So, currently a misbehaving initiator, not sending
	 * data in R2T state for a sharing between targets device, for which
	 * for some reason an aborting TM command, e.g. TARGET RESET, from
	 * another initiator is issued, can block response for this TM command
	 * virtually forever and by this make the issuing initiator eventually
	 * put the device offline.
	 *
	 * ToDo in the next version, possibly a simple connection mutex, taken
	 * by the read thread before starting any processing and by this
	 * function, should be sufficient.
	 */

	TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
		"ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
		"size to write %u, is_unsolicited_data %d, "
		"outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
		"conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
		atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
		cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
		cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
		cmnd->outstanding_r2t, cmnd->data_waiting,
		cmnd->conn->session->exp_cmd_sn, cmnd->conn,
		cmnd->conn->rd_task);

#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
	TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
#endif

	cmnd->tm_aborted = 1;

	return;
}

/* Must be called from the read or conn close thread */
static int cmnd_abort(struct iscsi_cmnd *req)
{
	struct iscsi_session *session = req->conn->session;
	struct iscsi_task_mgt_hdr *req_hdr =
		(struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
	struct iscsi_cmnd *cmnd;
	int err;

	req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);

	if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
		PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
			req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
		err = ISCSI_RESPONSE_FUNCTION_REJECTED;
		goto out;
	}

	cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG);
	if (cmnd) {
		struct iscsi_conn *conn = cmnd->conn;
		struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);

		if (req_hdr->lun != hdr->lun) {
			PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
				    "%llx, cmd LUN %llx, rtt %u",
				    (long long unsigned int)req_hdr->lun,
				    (long long unsigned int)hdr->lun,
				    req_hdr->rtt);
			err = ISCSI_RESPONSE_FUNCTION_REJECTED;
			goto out_put;
		}

		if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
			if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
				PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM "
					"cmd CmdSN(%u) for immediate command "
					"%p", req_hdr->ref_cmd_sn,
					req_hdr->cmd_sn, cmnd);
				err = ISCSI_RESPONSE_FUNCTION_REJECTED;
				goto out_put;
			}
		} else {
			if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
				PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
					"CmdSN(%u) for command %p",
					req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
					cmnd);
				err = ISCSI_RESPONSE_FUNCTION_REJECTED;
				goto out_put;
			}
		}

		if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
		    (req_hdr->cmd_sn == hdr->cmd_sn)) {
			PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
				"cmd SN %x, rtt %u", req_hdr->cmd_sn,
				hdr->cmd_sn, req_hdr->rtt);
			err = ISCSI_RESPONSE_FUNCTION_REJECTED;
			goto out_put;
		}

		spin_lock_bh(&conn->cmd_list_lock);
		__cmnd_abort(cmnd);
		spin_unlock_bh(&conn->cmd_list_lock);

		cmnd_put(cmnd);
		err = 0;
	} else {
		TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
		err = ISCSI_RESPONSE_UNKNOWN_TASK;
	}

out:
	return err;

out_put:
	cmnd_put(cmnd);
	goto out;
}

/* Must be called from the read or conn close thread */
static int target_abort(struct iscsi_cmnd *req, int all)
{
	struct iscsi_target *target = req->conn->session->target;
	struct iscsi_task_mgt_hdr *req_hdr =
		(struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
	struct iscsi_session *session;
	struct iscsi_conn *conn;
	struct iscsi_cmnd *cmnd;

	mutex_lock(&target->target_mutex);

	list_for_each_entry(session, &target->session_list,
			    session_list_entry) {
		list_for_each_entry(conn, &session->conn_list,
				    conn_list_entry) {
			spin_lock_bh(&conn->cmd_list_lock);
			list_for_each_entry(cmnd, &conn->cmd_list,
					    cmd_list_entry) {
				if (cmnd == req)
					continue;
				if (all)
					__cmnd_abort(cmnd);
				else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
					__cmnd_abort(cmnd);
			}
			spin_unlock_bh(&conn->cmd_list_lock);
		}
	}

	mutex_unlock(&target->target_mutex);
	return 0;
}

/* Must be called from the read or conn close thread */
static void task_set_abort(struct iscsi_cmnd *req)
{
	struct iscsi_session *session = req->conn->session;
	struct iscsi_task_mgt_hdr *req_hdr =
		(struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
	struct iscsi_target *target = session->target;
	struct iscsi_conn *conn;
	struct iscsi_cmnd *cmnd;

	mutex_lock(&target->target_mutex);

	list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
		spin_lock_bh(&conn->cmd_list_lock);
		list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
			struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
			if (cmnd == req)
				continue;
			if (req_hdr->lun != hdr->lun)
				continue;
			if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
			    req_hdr->cmd_sn == hdr->cmd_sn)
				continue;
			__cmnd_abort(cmnd);
		}
		spin_unlock_bh(&conn->cmd_list_lock);
	}

	mutex_unlock(&target->target_mutex);
	return;
}

/* Must be called from the read or conn close thread */
void conn_abort(struct iscsi_conn *conn)
{
	struct iscsi_cmnd *cmnd;

	TRACE_MGMT_DBG("Aborting conn %p", conn);

	iscsi_extracheck_is_rd_thread(conn);

	spin_lock_bh(&conn->cmd_list_lock);
again:
	list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
		__cmnd_abort(cmnd);
		if (cmnd->data_waiting) {
			if (!cmnd_get_check(cmnd)) {
				spin_unlock_bh(&conn->cmd_list_lock);

				/* ToDo: this is racy for MC/S */
				TRACE_MGMT_DBG("Restarting data waiting cmd "
					"%p", cmnd);
				iscsi_fail_waiting_cmnd(cmnd);

				cmnd_put(cmnd);

				/*
				 * We are in the read thread, so we may not
				 * worry that after cmnd release conn gets
				 * released as well.
				 */
				spin_lock_bh(&conn->cmd_list_lock);
				goto again;
			}
		}
	}
	spin_unlock_bh(&conn->cmd_list_lock);

	return;
}

static void execute_task_management(struct iscsi_cmnd *req)
{
	struct iscsi_conn *conn = req->conn;
	struct iscsi_session *sess = conn->session;
	struct iscsi_task_mgt_hdr *req_hdr =
		(struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
	int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
	struct scst_rx_mgmt_params params;

	TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ?
			TRACE_MGMT_MINOR : TRACE_MGMT,
		"TM fn %d", function);

	TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
		cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);

	iscsi_extracheck_is_rd_thread(conn);

	spin_lock(&sess->sn_lock);
	sess->tm_active++;
	sess->tm_sn = req_hdr->cmd_sn;
	if (sess->tm_rsp != NULL) {
		struct iscsi_cmnd *tm_rsp = sess->tm_rsp;

		TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);

		sess->tm_rsp = NULL;
		sess->tm_active--;

		spin_unlock(&sess->sn_lock);

		sBUG_ON(sess->tm_active < 0);

		rsp_cmnd_release(tm_rsp);
	} else
		spin_unlock(&sess->sn_lock);

	memset(&params, 0, sizeof(params));
	params.atomic = SCST_NON_ATOMIC;
	params.tgt_priv = req;

	if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
	    (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
		PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
			function);
		rc = -1;
		status = ISCSI_RESPONSE_FUNCTION_REJECTED;
		goto reject;
	}

	/* cmd_sn is already in CPU format converted in check_cmd_sn() */

	switch (function) {
	case ISCSI_FUNCTION_ABORT_TASK:
		rc = -1;
		status = cmnd_abort(req);
		if (status == 0) {
			params.fn = SCST_ABORT_TASK;
			params.tag = req_hdr->rtt;
			params.tag_set = 1;
			params.lun = (uint8_t *)&req_hdr->lun;
			params.lun_len = sizeof(req_hdr->lun);
			params.lun_set = 1;
			params.cmd_sn = req_hdr->cmd_sn;
			params.cmd_sn_set = 1;
			rc = scst_rx_mgmt_fn(conn->session->scst_sess,
				&params);
			status = ISCSI_RESPONSE_FUNCTION_REJECTED;
		}
		break;
	case ISCSI_FUNCTION_ABORT_TASK_SET:
		task_set_abort(req);
		params.fn = SCST_ABORT_TASK_SET;
		params.lun = (uint8_t *)&req_hdr->lun;
		params.lun_len = sizeof(req_hdr->lun);
		params.lun_set = 1;
		params.cmd_sn = req_hdr->cmd_sn;
		params.cmd_sn_set = 1;
		rc = scst_rx_mgmt_fn(conn->session->scst_sess,
			&params);
		status = ISCSI_RESPONSE_FUNCTION_REJECTED;
		break;
	case ISCSI_FUNCTION_CLEAR_TASK_SET:
		task_set_abort(req);
		params.fn = SCST_CLEAR_TASK_SET;
		params.lun = (uint8_t *)&req_hdr->lun;
		params.lun_len = sizeof(req_hdr->lun);
		params.lun_set = 1;
		params.cmd_sn = req_hdr->cmd_sn;
		params.cmd_sn_set = 1;
		rc = scst_rx_mgmt_fn(conn->session->scst_sess,
			&params);
		status = ISCSI_RESPONSE_FUNCTION_REJECTED;
		break;
	case ISCSI_FUNCTION_CLEAR_ACA:
		params.fn = SCST_CLEAR_ACA;
		params.lun = (uint8_t *)&req_hdr->lun;
		params.lun_len = sizeof(req_hdr->lun);
		params.lun_set = 1;
		params.cmd_sn = req_hdr->cmd_sn;
		params.cmd_sn_set = 1;
		rc = scst_rx_mgmt_fn(conn->session->scst_sess,
			&params);
		status = ISCSI_RESPONSE_FUNCTION_REJECTED;
		break;
	case ISCSI_FUNCTION_TARGET_COLD_RESET:
	case ISCSI_FUNCTION_TARGET_WARM_RESET:
		target_abort(req, 1);
		params.fn = SCST_TARGET_RESET;
		params.cmd_sn = req_hdr->cmd_sn;
		params.cmd_sn_set = 1;
		rc = scst_rx_mgmt_fn(conn->session->scst_sess,
			&params);
		status = ISCSI_RESPONSE_FUNCTION_REJECTED;
		break;
	case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
		target_abort(req, 0);
		params.fn = SCST_LUN_RESET;
		params.lun = (uint8_t *)&req_hdr->lun;
		params.lun_len = sizeof(req_hdr->lun);
		params.lun_set = 1;
		params.cmd_sn = req_hdr->cmd_sn;
		params.cmd_sn_set = 1;
		rc = scst_rx_mgmt_fn(conn->session->scst_sess,
			&params);
		status = ISCSI_RESPONSE_FUNCTION_REJECTED;
		break;
	case ISCSI_FUNCTION_TASK_REASSIGN:
		rc = -1;
		status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
		break;
	default:
		PRINT_ERROR("Unknown TM function %d", function);
		rc = -1;
		status = ISCSI_RESPONSE_FUNCTION_REJECTED;
		break;
	}

reject:
	if (rc != 0)
		iscsi_send_task_mgmt_resp(req, status);

	return;
}

static void noop_out_exec(struct iscsi_cmnd *req)
{
	struct iscsi_cmnd *rsp;
	struct iscsi_nop_in_hdr *rsp_hdr;

	TRACE_DBG("%p", req);

	if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
		rsp = iscsi_cmnd_create_rsp_cmnd(req);

		rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
		rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
		rsp_hdr->flags = ISCSI_FLG_FINAL;
		rsp_hdr->itt = req->pdu.bhs.itt;
		rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);

		if (req->pdu.datasize)
			sBUG_ON(req->sg == NULL);
		else
			sBUG_ON(req->sg != NULL);

		if (req->sg) {
			rsp->sg = req->sg;
			rsp->sg_cnt = req->sg_cnt;
			rsp->bufflen = req->bufflen;
		}

		/* We already checked it in check_segment_length() */
		sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);

		rsp->pdu.datasize = req->pdu.datasize;
		iscsi_cmnd_init_write(rsp,
			ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
		req_cmnd_release(req);
	} else
		cmnd_put(req);
	return;
}

static void logout_exec(struct iscsi_cmnd *req)
{
	struct iscsi_logout_req_hdr *req_hdr;
	struct iscsi_cmnd *rsp;
	struct iscsi_logout_rsp_hdr *rsp_hdr;

	PRINT_INFO("Logout received from initiator %s",
		req->conn->session->initiator_name);
	TRACE_DBG("%p", req);

	req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
	rsp = iscsi_cmnd_create_rsp_cmnd(req);
	rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
	rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
	rsp_hdr->flags = ISCSI_FLG_FINAL;
	rsp_hdr->itt = req_hdr->itt;
	rsp->should_close_conn = 1;
	iscsi_cmnd_init_write(rsp,
		ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
	req_cmnd_release(req);
	return;
}

static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
{
	TRACE_ENTRY();

	TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);

	iscsi_extracheck_is_rd_thread(cmnd->conn);

	if (unlikely(cmnd->tm_aborted)) {
		TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
			cmnd->scst_cmd);
		req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
		goto out;
	}

	if (unlikely(cmnd->rejected))
		goto out_rejected;

	switch (cmnd_opcode(cmnd)) {
	case ISCSI_OP_SCSI_CMD:
		if (cmnd->r2t_length != 0) {
			if (!cmnd->is_unsolicited_data) {
				send_r2t(cmnd);
				break;
			}
		} else
			iscsi_restart_cmnd(cmnd);
		break;
	case ISCSI_OP_NOOP_OUT:
		noop_out_exec(cmnd);
		break;
	case ISCSI_OP_SCSI_TASK_MGT_MSG:
		execute_task_management(cmnd);
		break;
	case ISCSI_OP_LOGOUT_CMD:
		logout_exec(cmnd);
		break;
	default:
		PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
		req_cmnd_release(cmnd);
		break;
	}
out:
	TRACE_EXIT();
	return;

out_rejected:
	TRACE_MGMT_DBG("Rejected cmd %p (reason %d)", cmnd,
		cmnd->reject_reason);
	switch (cmnd->reject_reason) {
	default:
		PRINT_ERROR("Unexpected reject reason %d",
			    cmnd->reject_reason);
		/* go through */
	case ISCSI_REJECT_SCSI_CMD:
		req_cmnd_release(cmnd);
		break;
	}
	goto out;
}

/*
 * Note: the code belows passes a kernel space pointer (&opt) to setsockopt()
 * while the declaration of setsockopt specifies that it expects a user space
 * pointer. This seems to work fine, and this approach is also used in some
 * other parts of the Linux kernel (see e.g. fs/ocfs2/cluster/tcp.c).
 */
static void set_cork(struct socket *sock, int on)
{
	int opt = on;
	mm_segment_t oldfs;

	oldfs = get_fs();
	set_fs(get_ds());
	sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
			      (void __force __user *)&opt, sizeof(opt));
	set_fs(oldfs);
	return;
}

void cmnd_tx_start(struct iscsi_cmnd *cmnd)
{
	struct iscsi_conn *conn = cmnd->conn;

	TRACE_DBG("conn %p, cmnd %p, opcode %x", conn, cmnd, cmnd_opcode(cmnd));
	iscsi_cmnd_set_length(&cmnd->pdu);

	iscsi_extracheck_is_wr_thread(conn);

	set_cork(conn->sock, 1);

	conn->write_iop = conn->write_iov;
	conn->write_iop->iov_base = (void __force __user *)(&cmnd->pdu.bhs);
	conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
	conn->write_iop_used = 1;
	conn->write_size = sizeof(cmnd->pdu.bhs) + cmnd->pdu.datasize;
	conn->write_offset = 0;

	switch (cmnd_opcode(cmnd)) {
	case ISCSI_OP_NOOP_IN:
		cmnd_set_sn(cmnd, 1);
		break;
	case ISCSI_OP_SCSI_RSP:
		cmnd_set_sn(cmnd, 1);
		break;
	case ISCSI_OP_SCSI_TASK_MGT_RSP:
		cmnd_set_sn(cmnd, 1);
		break;
	case ISCSI_OP_TEXT_RSP:
		cmnd_set_sn(cmnd, 1);
		break;
	case ISCSI_OP_SCSI_DATA_IN:
	{
		struct iscsi_data_in_hdr *rsp =
			(struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
		u32 offset = cpu_to_be32(rsp->buffer_offset);

		TRACE_DBG("cmnd %p, offset %u, datasize %u, bufflen %u", cmnd,
			offset, cmnd->pdu.datasize, cmnd->bufflen);

		sBUG_ON(offset > cmnd->bufflen);
		sBUG_ON(offset + cmnd->pdu.datasize > cmnd->bufflen);

		conn->write_offset = offset;

		cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
		break;
	}
	case ISCSI_OP_LOGOUT_RSP:
		cmnd_set_sn(cmnd, 1);
		break;
	case ISCSI_OP_R2T:
		cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
		break;
	case ISCSI_OP_ASYNC_MSG:
		cmnd_set_sn(cmnd, 1);
		break;
	case ISCSI_OP_REJECT:
		cmnd_set_sn(cmnd, 1);
		break;
	default:
		PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
		break;
	}

	iscsi_dump_pdu(&cmnd->pdu);
	return;
}

void cmnd_tx_end(struct iscsi_cmnd *cmnd)
{
	struct iscsi_conn *conn = cmnd->conn;

	TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
		cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
		cmnd->should_close_all_conn);

	switch (cmnd_opcode(cmnd)) {
	case ISCSI_OP_NOOP_IN:
	case ISCSI_OP_SCSI_RSP:
	case ISCSI_OP_SCSI_TASK_MGT_RSP:
	case ISCSI_OP_TEXT_RSP:
	case ISCSI_OP_R2T:
	case ISCSI_OP_ASYNC_MSG:
	case ISCSI_OP_REJECT:
	case ISCSI_OP_SCSI_DATA_IN:
	case ISCSI_OP_LOGOUT_RSP:
		break;
	default:
		PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
		sBUG();
		break;
	}

	if (unlikely(cmnd->should_close_conn)) {
		if (cmnd->should_close_all_conn) {
			PRINT_INFO("Closing all connections for target %x at "
				"initiator's %s request",
				cmnd->conn->session->target->tid,
				conn->session->initiator_name);
			target_del_all_sess(cmnd->conn->session->target, 0);
		} else {
			PRINT_INFO("Closing connection at initiator's %s "
				"request", conn->session->initiator_name);
			mark_conn_closed(conn);
		}
	}

	set_cork(cmnd->conn->sock, 0);
	return;
}

/*
 * Push the command for execution. This functions reorders the commands.
 * Called from the read thread.
 */
static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
{
	struct iscsi_session *session = cmnd->conn->session;
	struct list_head *entry;
	u32 cmd_sn;

	TRACE_DBG("%p:%x %u,%u",
		cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn,
		session->exp_cmd_sn);

	iscsi_extracheck_is_rd_thread(cmnd->conn);

	sBUG_ON(cmnd->parent_req != NULL);

	if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
		TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
			cmnd->pdu.bhs.sn);
		iscsi_cmnd_exec(cmnd);
		goto out;
	}

	spin_lock(&session->sn_lock);

	cmd_sn = cmnd->pdu.bhs.sn;
	if (cmd_sn == session->exp_cmd_sn) {
		while (1) {
			session->exp_cmd_sn = ++cmd_sn;

			if (unlikely(session->tm_active > 0)) {
				if (before(cmd_sn, session->tm_sn)) {
					struct iscsi_conn *conn = cmnd->conn;

					spin_unlock(&session->sn_lock);

					spin_lock_bh(&conn->cmd_list_lock);
					__cmnd_abort(cmnd);
					spin_unlock_bh(&conn->cmd_list_lock);

					spin_lock(&session->sn_lock);
				}
				iscsi_check_send_delayed_tm_resp(session);
			}

			spin_unlock(&session->sn_lock);

			iscsi_cmnd_exec(cmnd);

			spin_lock(&session->sn_lock);

			if (list_empty(&session->pending_list))
				break;
			cmnd = list_entry(session->pending_list.next,
					  struct iscsi_cmnd,
					  pending_list_entry);
			if (cmnd->pdu.bhs.sn != cmd_sn)
				break;

			list_del(&cmnd->pending_list_entry);
			cmnd->pending = 0;

			TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
				cmnd, cmd_sn);
		}
	} else {
		int drop = 0;

		TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
			cmnd, cmd_sn, session->exp_cmd_sn);

		/*
		 * iSCSI RFC 3720: "The target MUST silently ignore any
		 * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
		 * inclusive] range". But we won't honor the MaxCmdSN
		 * requirement, because, since we adjust MaxCmdSN from the
		 * separate write thread, rarery it is possible that initiator
		 * can legally send command with CmdSN>MaxSN. But it won't
		 * hurt anything, in the worst case it will lead to
		 * additional QUEUE FULL status.
		 */

		if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
			PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
				session->exp_cmd_sn);
			drop = 1;
		}

#if 0
		if (unlikely(after(cmd_sn, session->exp_cmd_sn +
					iscsi_get_allowed_cmds(session)))) {
			TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
				"max_sn %u)", cmd_sn, session->exp_cmd_sn,
				iscsi_get_allowed_cmds(session));
		}
#endif

		spin_unlock(&session->sn_lock);

		if (unlikely(drop)) {
			req_cmnd_release_force(cmnd,
					       ISCSI_FORCE_RELEASE_WRITE);
			goto out;
		}

		if (unlikely(cmnd->tm_aborted)) {
			struct iscsi_cmnd *tm_clone;

			TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
				"clone (scst cmd %p, state %d)", cmnd,
				cmnd->scst_cmd, cmnd->scst_state);

			tm_clone = cmnd_alloc(cmnd->conn, NULL);
			if (tm_clone != NULL) {
				tm_clone->tm_aborted = 1;
				tm_clone->pdu = cmnd->pdu;

				TRACE_MGMT_DBG("TM clone %p created",
					       tm_clone);

				iscsi_cmnd_exec(cmnd);
				cmnd = tm_clone;
			} else
				PRINT_ERROR("%s", "Unable to create TM clone");
		}

		spin_lock(&session->sn_lock);
		list_for_each(entry, &session->pending_list) {
			struct iscsi_cmnd *tmp =
				list_entry(entry, struct iscsi_cmnd,
					   pending_list_entry);
			if (before(cmd_sn, tmp->pdu.bhs.sn))
				break;
		}
		list_add_tail(&cmnd->pending_list_entry, entry);
		cmnd->pending = 1;
	}

	spin_unlock(&session->sn_lock);
out:
	return;
}

static int check_segment_length(struct iscsi_cmnd *cmnd)
{
	struct iscsi_conn *conn = cmnd->conn;
	struct iscsi_session *session = conn->session;

	if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
		PRINT_ERROR("Initiator %s violated negotiated parameters: "
			"data too long (ITT %x, datasize %u, "
			"max_recv_data_length %u", session->initiator_name,
			cmnd_itt(cmnd), cmnd->pdu.datasize,
			session->sess_param.max_recv_data_length);
		mark_conn_closed(conn);
		return -EINVAL;
	}
	return 0;
}

int cmnd_rx_start(struct iscsi_cmnd *cmnd)
{
	struct iscsi_conn *conn = cmnd->conn;
	int res, rc;

	iscsi_dump_pdu(&cmnd->pdu);

	res = check_segment_length(cmnd);
	if (res != 0)
		goto out;

	switch (cmnd_opcode(cmnd)) {
	case ISCSI_OP_NOOP_OUT:
		rc = noop_out_start(cmnd);
		break;
	case ISCSI_OP_SCSI_CMD:
		rc = cmnd_insert_hash(cmnd);
		if (likely(rc == 0)) {
			res = scsi_cmnd_start(cmnd);
			if (unlikely(res != 0))
				goto out;
		}
		break;
	case ISCSI_OP_SCSI_TASK_MGT_MSG:
		rc = cmnd_insert_hash(cmnd);
		break;
	case ISCSI_OP_SCSI_DATA_OUT:
		res = data_out_start(conn, cmnd);
		rc = 0; /* to avoid compiler warning */
		if (unlikely(res != 0))
			goto out;
		break;
	case ISCSI_OP_LOGOUT_CMD:
		rc = cmnd_insert_hash(cmnd);
		break;
	case ISCSI_OP_TEXT_CMD:
	case ISCSI_OP_SNACK_CMD:
		rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
		break;
	default:
		rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
		break;
	}

	if (unlikely(rc < 0)) {
		struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
		PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
			cmnd_opcode(cmnd), cmnd_itt(cmnd),
			(cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
				hdr->scb[0] : -1));
		iscsi_cmnd_reject(cmnd, -rc);
	}

out:
	TRACE_EXIT_RES(res);
	return res;
}

void cmnd_rx_end(struct iscsi_cmnd *cmnd)
{
	TRACE_ENTRY();

	TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));

	if (unlikely(cmnd->rejected))
		goto out_rejected;

cont:
	switch (cmnd_opcode(cmnd)) {
	case ISCSI_OP_SCSI_CMD:
	case ISCSI_OP_NOOP_OUT:
	case ISCSI_OP_SCSI_TASK_MGT_MSG:
	case ISCSI_OP_LOGOUT_CMD:
		iscsi_session_push_cmnd(cmnd);
		break;
	case ISCSI_OP_SCSI_DATA_OUT:
		data_out_end(cmnd);
		break;
	default:
		PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
		req_cmnd_release(cmnd);
		break;
	}

out:
	TRACE_EXIT();
	return;

out_rejected:
	switch (cmnd->reject_reason) {
	default:
		PRINT_ERROR("Unexpected reject reason %d",
			    cmnd->reject_reason);
		/* go through */
	case ISCSI_REJECT_CMD:
	case ISCSI_REJECT_DATA:
		req_cmnd_release(cmnd);
		break;
	case ISCSI_REJECT_SCSI_CMD:
		goto cont;
	}
	goto out;
}

#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
{
	/*
	 * sock->ops->sendpage() is async zero copy operation,
	 * so we must be sure not to free and reuse
	 * the command's buffer before the sending was completed
	 * by the network layers. It is possible only if we
	 * don't use SGV cache.
	 */
	EXTRACHECKS_BUG_ON(scst_cmd_get_data_direction(cmd) != SCST_DATA_READ);
	scst_cmd_set_no_sgv(cmd);
	return 1;
}
#endif

static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
	int new_state)
{
	/*
	 * We use wait_event() to wait for the state change, but it checks its
	 * condition without any protection, so without cmnd_get() it is
	 * possible that req will die "immediately" after the state assignment
	 * and wake_up() will operate on dead data. We use the ordered version
	 * of cmnd_get(), because "get" must be done before the state
	 * assignment.
	 */
	cmnd_get_ordered(req);
	req->scst_state = new_state;
	wake_up(&req->scst_waitQ);
	cmnd_put(req);
	return;
}

static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
{
	struct iscsi_cmnd *req = (struct iscsi_cmnd *)
				scst_cmd_get_tgt_priv(scst_cmd);

	TRACE_DBG("req %p", req);

	EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);

	iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
	return;
}

/*
 * No locks.
 *
 * IMPORTANT! Connection conn must be protected by additional conn_get()
 * upon entrance in this function, because otherwise it could be destroyed
 * inside as a result of iscsi_send(), which releases sent commands.
 */
static void iscsi_try_local_processing(struct iscsi_conn *conn)
{
	int local;

	TRACE_ENTRY();

	spin_lock_bh(&iscsi_wr_lock);
	switch (conn->wr_state) {
	case ISCSI_CONN_WR_STATE_IN_LIST:
		list_del(&conn->wr_list_entry);
		/* go through */
	case ISCSI_CONN_WR_STATE_IDLE:
#ifdef CONFIG_SCST_EXTRACHECKS
		conn->wr_task = current;
#endif
		conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
		conn->wr_space_ready = 0;
		local = 1;
		break;
	default:
		local = 0;
		break;
	}
	spin_unlock_bh(&iscsi_wr_lock);

	if (local) {
		int rc = 1;

		if (test_write_ready(conn))
			rc = iscsi_send(conn);

		spin_lock_bh(&iscsi_wr_lock);
#ifdef CONFIG_SCST_EXTRACHECKS
		conn->wr_task = NULL;
#endif
		if ((rc <= 0) || test_write_ready(conn)) {
			list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
			conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
			wake_up(&iscsi_wr_waitQ);
		} else
			conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
		spin_unlock_bh(&iscsi_wr_lock);
	}

	TRACE_EXIT();
	return;
}

static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
{
	int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
	struct iscsi_cmnd *req = (struct iscsi_cmnd *)
					scst_cmd_get_tgt_priv(scst_cmd);
	struct iscsi_conn *conn = req->conn;
	int status = scst_cmd_get_status(scst_cmd);
	u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
	int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
	int old_state = req->scst_state;

	if (scst_cmd_atomic(scst_cmd))
		return SCST_TGT_RES_NEED_THREAD_CTX;

	scst_cmd_set_tgt_priv(scst_cmd, NULL);

	req->tm_aborted |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
	if (unlikely(req->tm_aborted)) {
		TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
			req->scst_cmd);

		scst_set_delivery_status(req->scst_cmd,
			SCST_CMD_DELIVERY_ABORTED);

		if (old_state == ISCSI_CMD_STATE_RESTARTED) {
			req->scst_state = ISCSI_CMD_STATE_PROCESSED;
			req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
		} else
			iscsi_set_state_wake_up(req,
						ISCSI_CMD_STATE_PROCESSED);

		goto out;
	}

	if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
		TRACE_DBG("req %p on %d state", req, old_state);

		create_status_rsp(req, status, sense, sense_len);

		switch (old_state) {
		case ISCSI_CMD_STATE_RX_CMD:
		case ISCSI_CMD_STATE_AFTER_PREPROC:
			break;
		default:
			sBUG();
		}

		iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
		goto out;
	}

	req->scst_state = ISCSI_CMD_STATE_PROCESSED;

	req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
	req->sg = scst_cmd_get_sg(scst_cmd);
	req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);

	TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
		"req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
		req->sg_cnt);

	if (unlikely((req->bufflen != 0) && !is_send_status)) {
		PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
			"unsupported");
		scst_set_cmd_error(scst_cmd,
			SCST_LOAD_SENSE(scst_sense_hardw_error));
		sBUG();
	}

	if (req->bufflen != 0) {
		/*
		 * Check above makes sure that is_send_status is set,
		 * so status is valid here, but in future that could change.
		 * ToDo
		 */
		if (status != SAM_STAT_CHECK_CONDITION) {
			send_data_rsp(req, status, is_send_status);
		} else {
			struct iscsi_cmnd *rsp;
			struct iscsi_scsi_rsp_hdr *rsp_hdr;
			int resid;
			send_data_rsp(req, 0, 0);
			if (is_send_status) {
				rsp = create_status_rsp(req, status, sense,
					sense_len);
				rsp_hdr =
				    (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
				resid = cmnd_read_size(req) - req->bufflen;
				if (resid > 0) {
					rsp_hdr->flags |=
						ISCSI_FLG_RESIDUAL_UNDERFLOW;
					rsp_hdr->residual_count =
						cpu_to_be32(resid);
				} else if (resid < 0) {
					rsp_hdr->flags |=
						ISCSI_FLG_RESIDUAL_OVERFLOW;
					rsp_hdr->residual_count =
						cpu_to_be32(-resid);
				}
				iscsi_cmnd_init_write(rsp,
					ISCSI_INIT_WRITE_REMOVE_HASH);
			}
		}
	} else if (is_send_status) {
		struct iscsi_cmnd *rsp;
		struct iscsi_scsi_rsp_hdr *rsp_hdr;
		u32 resid;
		rsp = create_status_rsp(req, status, sense, sense_len);
		rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
		resid = cmnd_read_size(req);
		if (resid != 0) {
			rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
			rsp_hdr->residual_count = cpu_to_be32(resid);
		}
		iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
	}
#ifdef CONFIG_SCST_EXTRACHECKS
	else
		sBUG();
#endif

	/*
	 * "_ordered" here to protect from reorder, which can lead to
	 * preliminary connection destroy in req_cmnd_release(). Just in
	 * case, actually, because reordering shouldn't go so far, but who
	 * knows..
	 */
	conn_get_ordered(conn);
	req_cmnd_release(req);
	iscsi_try_local_processing(conn);
	conn_put(conn);

out:
	return SCST_TGT_RES_SUCCESS;
}

/* Called under sn_lock */
static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
{
	bool res = 0;
	struct iscsi_task_mgt_hdr *req_hdr =
		(struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
	int function = req_hdr->function & ISCSI_FUNCTION_MASK;
	struct iscsi_session *sess = rsp->conn->session;

	TRACE_ENTRY();

	/* This should be checked for immediate TM commands as well */

	switch (function) {
	default:
		if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
			res = 1;
		break;
	}

	TRACE_EXIT_RES(res);
	return res;
}

/* Called under sn_lock, but might drop it inside, then reaquire */
static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
{
	struct iscsi_cmnd *tm_rsp = sess->tm_rsp;

	TRACE_ENTRY();

	if (tm_rsp == NULL)
		goto out;

	if (iscsi_is_delay_tm_resp(tm_rsp))
		goto out;

	TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);

	sess->tm_rsp = NULL;
	sess->tm_active--;

	spin_unlock(&sess->sn_lock);

	sBUG_ON(sess->tm_active < 0);

	iscsi_cmnd_init_write(tm_rsp,
		ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);

	spin_lock(&sess->sn_lock);

out:
	TRACE_EXIT();
	return;
}

static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
{
	struct iscsi_cmnd *rsp;
	struct iscsi_task_mgt_hdr *req_hdr =
				(struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
	struct iscsi_task_rsp_hdr *rsp_hdr;
	struct iscsi_session *sess = req->conn->session;
	int fn = req_hdr->function & ISCSI_FUNCTION_MASK;

	TRACE_ENTRY();

	TRACE_MGMT_DBG("TM req %p finished", req);
	TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
			 TRACE_MGMT_MINOR : TRACE_MGMT,
		"TM fn %d finished, status %d", fn, status);

	rsp = iscsi_cmnd_create_rsp_cmnd(req);
	rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;

	rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
	rsp_hdr->flags = ISCSI_FLG_FINAL;
	rsp_hdr->itt = req_hdr->itt;
	rsp_hdr->response = status;

	if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
		rsp->should_close_conn = 1;
		rsp->should_close_all_conn = 1;
	}

	sBUG_ON(sess->tm_rsp != NULL);

	spin_lock(&sess->sn_lock);
	if (iscsi_is_delay_tm_resp(rsp)) {
		TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
			"(req %p), because not all affected commands received "
			"(TM cmd sn %u, exp sn %u)",
			req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
			req_hdr->cmd_sn, sess->exp_cmd_sn);
		sess->tm_rsp = rsp;
		spin_unlock(&sess->sn_lock);
		goto out_release;
	}
	sess->tm_active--;
	spin_unlock(&sess->sn_lock);

	sBUG_ON(sess->tm_active < 0);

	iscsi_cmnd_init_write(rsp,
		ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);

out_release:
	req_cmnd_release(req);

	TRACE_EXIT();
	return;
}

static inline int iscsi_get_mgmt_response(int status)
{
	switch (status) {
	case SCST_MGMT_STATUS_SUCCESS:
		return ISCSI_RESPONSE_FUNCTION_COMPLETE;

	case SCST_MGMT_STATUS_TASK_NOT_EXIST:
		return ISCSI_RESPONSE_UNKNOWN_TASK;

	case SCST_MGMT_STATUS_LUN_NOT_EXIST:
		return ISCSI_RESPONSE_UNKNOWN_LUN;

	case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
		return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;

	case SCST_MGMT_STATUS_REJECTED:
	case SCST_MGMT_STATUS_FAILED:
	default:
		return ISCSI_RESPONSE_FUNCTION_REJECTED;
	}
}

static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
{
	int fn = scst_mgmt_cmd_get_fn(scst_mcmd);
	struct iscsi_cmnd *req = (struct iscsi_cmnd *)
				scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
	int status =
		iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));

	TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
		req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd));

	switch (fn) {
	case SCST_NEXUS_LOSS_SESS:
	case SCST_ABORT_ALL_TASKS_SESS:
		/* They are internal */
		break;
	default:
		iscsi_send_task_mgmt_resp(req, status);
		scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
		break;
	}
	return;
}

static int iscsi_scsi_aen(struct scst_aen *aen)
{
	int res = SCST_AEN_RES_SUCCESS;
	uint64_t lun = scst_aen_get_lun(aen);
	const uint8_t *sense = scst_aen_get_sense(aen);
	int sense_len = scst_aen_get_sense_len(aen);
	struct iscsi_session *sess = scst_sess_get_tgt_priv(
					scst_aen_get_sess(aen));
	struct iscsi_conn *conn;
	bool found;
	struct iscsi_cmnd *fake_req, *rsp;
	struct iscsi_async_msg_hdr *rsp_hdr;
	struct scatterlist *sg;

	TRACE_ENTRY();

	TRACE_MGMT_DBG("SCSI AEN to sess %p (initiator %s)", sess,
		sess->initiator_name);

	mutex_lock(&sess->target->target_mutex);

	found = false;
	list_for_each_entry_reverse(conn, &sess->conn_list, conn_list_entry) {
		if (!conn->conn_shutting_down &&
		    (conn->conn_reinst_successor == NULL)) {
			found = true;
			break;
		}
	}
	if (!found) {
		TRACE_MGMT_DBG("Unable to find alive conn for sess %p", sess);
		goto out_err;
	}

	/* Create a fake request */
	fake_req = cmnd_alloc(conn, NULL);
	if (fake_req == NULL) {
		PRINT_ERROR("%s", "Unable to alloc fake AEN request");
		goto out_err;
	}

	mutex_unlock(&sess->target->target_mutex);

	rsp = iscsi_cmnd_create_rsp_cmnd(fake_req);
	if (rsp == NULL) {
		PRINT_ERROR("%s", "Unable to alloc AEN rsp");
		goto out_err_free_req;
	}

	fake_req->scst_state = ISCSI_CMD_STATE_AEN;
	fake_req->scst_aen = aen;

	rsp_hdr = (struct iscsi_async_msg_hdr *)&rsp->pdu.bhs;

	rsp_hdr->opcode = ISCSI_OP_ASYNC_MSG;
	rsp_hdr->flags = ISCSI_FLG_FINAL;
	rsp_hdr->lun = lun; /* it's already in SCSI form */
	rsp_hdr->ffffffff = 0xffffffff;
	rsp_hdr->async_event = ISCSI_ASYNC_SCSI;

	sg = rsp->sg = rsp->rsp_sg;
	rsp->sg_cnt = 2;
	rsp->own_sg = 1;

	sg_init_table(sg, 2);
	sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
	sg_set_buf(&sg[1], sense, sense_len);

	rsp->sense_hdr.length = cpu_to_be16(sense_len);
	rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
	rsp->bufflen = rsp->pdu.datasize;

	iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);

	req_cmnd_release(fake_req);

out:
	TRACE_EXIT_RES(res);
	return res;

out_err_free_req:
	req_cmnd_release(fake_req);

out_err:
	mutex_unlock(&sess->target->target_mutex);
	res = SCST_AEN_RES_FAILED;
	goto out;
}

static int iscsi_report_aen(struct scst_aen *aen)
{
	int res;
	int event_fn = scst_aen_get_event_fn(aen);

	TRACE_ENTRY();

	switch (event_fn) {
	case SCST_AEN_SCSI:
		res = iscsi_scsi_aen(aen);
		break;
	default:
		TRACE_MGMT_DBG("Unsupported AEN %d", event_fn);
		res = SCST_AEN_RES_NOT_SUPPORTED;
		break;
	}

	TRACE_EXIT_RES(res);
	return res;
}

static int iscsi_target_detect(struct scst_tgt_template *templ)
{
	/* Nothing to do */
	return 0;
}

static int iscsi_target_release(struct scst_tgt *scst_tgt)
{
	/* Nothing to do */
	return 0;
}

struct scst_tgt_template iscsi_template = {
	.name = "iscsi",
	.sg_tablesize = 0xFFFF /* no limit */,
	.threads_num = 0,
	.no_clustering = 1,
	.xmit_response_atomic = 0,
	.detect = iscsi_target_detect,
	.release = iscsi_target_release,
	.xmit_response = iscsi_xmit_response,
#if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
	.alloc_data_buf = iscsi_alloc_data_buf,
#endif
	.preprocessing_done = iscsi_preprocessing_done,
	.pre_exec = iscsi_pre_exec,
	.task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done,
	.task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
	.report_aen = iscsi_report_aen,
};

static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
{
	int res = 0;
	int i;
	struct iscsi_thread_t *thr;

	for (i = 0; i < count; i++) {
		thr = kmalloc(sizeof(*thr), GFP_KERNEL);
		if (!thr) {
			res = -ENOMEM;
			PRINT_ERROR("Failed to allocate thr %d", res);
			goto out;
		}
		thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
		if (IS_ERR(thr->thr)) {
			res = PTR_ERR(thr->thr);
			PRINT_ERROR("kthread_create() failed: %d", res);
			kfree(thr);
			goto out;
		}
		list_add_tail(&thr->threads_list_entry, &iscsi_threads_list);
	}

out:
	return res;
}

static void iscsi_stop_threads(void)
{
	struct iscsi_thread_t *t, *tmp;

	list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
				threads_list_entry) {
		int rc = kthread_stop(t->thr);
		if (rc < 0)
			TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
		list_del(&t->threads_list_entry);
		kfree(t);
	}
	return;
}

static int __init iscsi_init(void)
{
	int err = 0;
	int num;

	PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);

	sense_fixed_unexpected_unsolicited_data[0] = 0x70;
	sense_fixed_unexpected_unsolicited_data[2] = ABORTED_COMMAND;
	sense_fixed_unexpected_unsolicited_data[7] = 6;
	sense_fixed_unexpected_unsolicited_data[12] = 0xc;
	sense_fixed_unexpected_unsolicited_data[13] = 0xc;

	sense_descr_unexpected_unsolicited_data[0] = 0x72;
	sense_descr_unexpected_unsolicited_data[1] = ABORTED_COMMAND;
	sense_descr_unexpected_unsolicited_data[2] = 0xc;
	sense_descr_unexpected_unsolicited_data[3] = 0xc;

	dummy_page = alloc_pages(GFP_KERNEL, 0);
	if (dummy_page == NULL) {
		PRINT_ERROR("%s", "Dummy page allocation failed");
		goto out;
	}

	sg_init_table(&dummy_sg, 1);
	sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);

#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
	err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
			iscsi_put_page_callback);
	if (err != 0) {
		PRINT_INFO("Unable to set page callbackes: %d", err);
		goto out_free_dummy;
	}
#else
	PRINT_WARNING("%s",
		"CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION "
		"not enabled in your kernel. ISCSI-SCST will be working with "
		"not the best performance. Refer README file for details.");
#endif

	ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
	if (ctr_major < 0) {
		PRINT_ERROR("failed to register the control device %d",
			    ctr_major);
		err = ctr_major;
		goto out_callb;
	}

	err = event_init();
	if (err < 0)
		goto out_reg;

	iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
	if (!iscsi_cmnd_cache) {
		err = -ENOMEM;
		goto out_event;
	}

	err = scst_register_target_template(&iscsi_template);
	if (err < 0)
		goto out_kmem;

	iscsi_template_registered = 1;

	err = iscsi_procfs_init();
	if (err < 0)
		goto out_reg_tmpl;

	num = max((int)num_online_cpus(), 2);

	err = iscsi_run_threads(num, "iscsird", istrd);
	if (err != 0)
		goto out_thr;

	err = iscsi_run_threads(num, "iscsiwr", istwr);
	if (err != 0)
		goto out_thr;

out:
	return err;

out_thr:
	iscsi_procfs_exit();
	iscsi_stop_threads();

out_reg_tmpl:
	scst_unregister_target_template(&iscsi_template);

out_kmem:
	kmem_cache_destroy(iscsi_cmnd_cache);

out_event:
	event_exit();

out_reg:
	unregister_chrdev(ctr_major, ctr_name);

out_callb:
#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
	net_set_get_put_page_callbacks(NULL, NULL);

out_free_dummy:
#endif
	__free_pages(dummy_page, 0);
	goto out;
}

static void __exit iscsi_exit(void)
{
	iscsi_stop_threads();

	unregister_chrdev(ctr_major, ctr_name);

	iscsi_procfs_exit();
	event_exit();

	kmem_cache_destroy(iscsi_cmnd_cache);

	scst_unregister_target_template(&iscsi_template);

#if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
	net_set_get_put_page_callbacks(NULL, NULL);
#endif

	__free_pages(dummy_page, 0);
	return;
}

module_init(iscsi_init);
module_exit(iscsi_exit);

MODULE_LICENSE("GPL");