mirrors/scst
1
0
Fork 0
mirror of https://github.com/SCST-project/scst.git synced 2026-05-14 01:01:27 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
scst/iscsi-scst/kernel/iscsi.c
Gleb Chesnokov 8a3b257c33 scst: Replace kmalloc with kmalloc_obj for non-scalar types 
This is the result of running the Linux kernel Coccinelle script from
scripts/coccinelle/api/kmalloc_objs.cocci against the SCST tree.

This patch doesn't change any functionality.
2026-04-12 15:38:24 +03:00

4304 lines
110 KiB
C
Raw Permalink Blame History

/*
* Copyright (C) 2002 - 2003 Ardis Technologies <roman@ardistech.com>
* Copyright (C) 2007 - 2018 Vladislav Bolkhovitin
* Copyright (C) 2007 - 2018 Western Digital Corporation
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation, version 2
* of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*/
#include <linux/module.h>
#include <linux/hash.h>
#include <linux/kthread.h>
#include <linux/scatterlist.h>
#include <linux/ctype.h>
#include <net/tcp.h>
#include <scsi/scsi.h>
#include <asm/byteorder.h>
#ifdef INSIDE_KERNEL_TREE
#include <scst/iscsit_transport.h>
#else
#include "iscsit_transport.h"
#endif
#include "iscsi_trace_flag.h"
#include "iscsi.h"
#include "digest.h"
#undef DEFAULT_SYMBOL_NAMESPACE
#define DEFAULT_SYMBOL_NAMESPACE SCST_NAMESPACE
#define ISCSI_INIT_WRITE_WAKE 0x1
static int ctr_major;
static const char ctr_name[] = "iscsi-scst-ctl";
#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
#endif
static struct kmem_cache *iscsi_cmnd_cache;
static DEFINE_MUTEX(iscsi_threads_pool_mutex);
static LIST_HEAD(iscsi_thread_pools_list);
static struct kmem_cache *iscsi_thread_pool_cache;
static struct iscsi_thread_pool *iscsi_main_thread_pool;
struct kmem_cache *iscsi_conn_cache;
struct kmem_cache *iscsi_sess_cache;
static struct page *dummy_page;
static struct scatterlist dummy_sg[1];
static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd);
static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status, bool dropped);
static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
static void req_cmnd_release(struct iscsi_cmnd *req);
static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd);
static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags);
static void iscsi_set_resid_no_scst_cmd(struct iscsi_cmnd *rsp);
static void iscsi_set_not_received_data_len(struct iscsi_cmnd *req, unsigned int not_received)
{
req->not_received_data_len = not_received;
if (req->scst_cmd)
scst_cmd_set_write_not_received_data_len(req->scst_cmd, not_received);
}
static void req_del_from_write_timeout_list(struct iscsi_cmnd *req)
{
struct iscsi_conn *conn;
TRACE_ENTRY();
if (!req->on_write_timeout_list)
goto out;
conn = req->conn;
TRACE_DBG("Deleting cmd %p from conn %p write_timeout_list",
req, conn);
spin_lock_bh(&conn->write_list_lock);
/* Recheck, since it can be changed behind us */
if (unlikely(!req->on_write_timeout_list))
goto out_unlock;
list_del(&req->write_timeout_list_entry);
req->on_write_timeout_list = 0;
out_unlock:
spin_unlock_bh(&conn->write_list_lock);
out:
TRACE_EXIT();
}
static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
{
struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
if (hdr->flags & ISCSI_CMD_WRITE)
return be32_to_cpu(hdr->data_length);
return 0;
}
static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
{
struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
if (hdr->flags & ISCSI_CMD_READ) {
struct iscsi_ahs_hdr *ahdr;
if (!(hdr->flags & ISCSI_CMD_WRITE))
return be32_to_cpu(hdr->data_length);
ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
if (ahdr) {
uint8_t *p = (uint8_t *)ahdr;
unsigned int size = 0;
do {
int s;
ahdr = (struct iscsi_ahs_hdr *)p;
if (ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH) {
struct iscsi_rlength_ahdr *rh =
(struct iscsi_rlength_ahdr *)ahdr;
return be32_to_cpu(rh->read_length);
}
s = 3 + be16_to_cpu(ahdr->ahslength);
s = (s + 3) & -4;
size += s;
p += s;
} while (size < cmnd->pdu.ahssize);
}
return -1;
}
return 0;
}
void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
{
int status;
TRACE_ENTRY();
EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_receive != 0);
EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_send != 0);
req_del_from_write_timeout_list(cmnd);
/*
* Let's remove cmnd from the hash earlier to keep it smaller.
* Also we have to remove hashed req from the hash before sending
* response. Otherwise we can have a race, when for some reason cmd's
* release (and, hence, removal from the hash) is delayed after the
* transmission and initiator sends cmd with the same ITT, hence
* the new command will be erroneously rejected as a duplicate.
*/
if (cmnd->hashed)
cmnd_remove_data_wait_hash(cmnd);
if (unlikely(test_bit(ISCSI_CONN_REINSTATING, &cmnd->conn->conn_aflags))) {
struct iscsi_target *target = cmnd->conn->session->target;
bool get_out;
mutex_lock(&target->target_mutex);
get_out = test_bit(ISCSI_CONN_REINSTATING, &cmnd->conn->conn_aflags);
/* Let's don't look dead */
if (scst_cmd_get_cdb(cmnd->scst_cmd)[0] == TEST_UNIT_READY)
get_out = false;
if (!get_out)
goto unlock_cont;
TRACE_MGMT_DBG("Pending cmnd %p, because conn %p is reinstated",
cmnd, cmnd->conn);
cmnd->scst_state = ISCSI_CMD_STATE_REINST_PENDING;
list_add_tail(&cmnd->reinst_pending_cmd_list_entry,
&cmnd->conn->reinst_pending_cmd_list);
unlock_cont:
mutex_unlock(&target->target_mutex);
if (get_out)
goto out;
}
if (unlikely(cmnd->prelim_compl_flags != 0)) {
if (test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)) {
TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted",
cmnd, cmnd->scst_cmd);
req_cmnd_release_force(cmnd);
goto out;
}
if (!cmnd->scst_cmd) {
TRACE_MGMT_DBG("Finishing preliminary completed cmd %p with NULL scst_cmd",
cmnd);
req_cmnd_release(cmnd);
goto out;
}
status = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
} else {
status = SCST_PREPROCESS_STATUS_SUCCESS;
}
cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
scst_restart_cmd(cmnd->scst_cmd, status, SCST_CONTEXT_THREAD);
out:
TRACE_EXIT();
}
static struct iscsi_cmnd *iscsi_create_tm_clone(struct iscsi_cmnd *cmnd)
{
struct iscsi_cmnd *tm_clone;
TRACE_ENTRY();
tm_clone = cmnd->conn->transport->iscsit_alloc_cmd(cmnd->conn, NULL);
if (tm_clone) {
set_bit(ISCSI_CMD_ABORTED, &tm_clone->prelim_compl_flags);
tm_clone->pdu = cmnd->pdu;
TRACE_MGMT_DBG("TM clone %p for cmnd %p created",
tm_clone, cmnd);
} else {
PRINT_ERROR("Failed to create TM clone for cmnd %p", cmnd);
}
TRACE_EXIT_HRES(tm_clone);
return tm_clone;
}
void iscsi_fail_data_waiting_cmnd(struct iscsi_cmnd *cmnd)
{
TRACE_ENTRY();
TRACE_MGMT_DBG("Failing data waiting cmnd %p (data_out_in_data_receiving %d)",
cmnd, cmnd->data_out_in_data_receiving);
/*
* There is no race with conn_abort(), since all functions
* called from single read thread
*/
iscsi_extracheck_is_rd_thread(cmnd->conn);
/* This cmnd is going to die without response */
cmnd->r2t_len_to_receive = 0;
cmnd->r2t_len_to_send = 0;
if (cmnd->pending) {
struct iscsi_session *session = cmnd->conn->session;
struct iscsi_cmnd *tm_clone;
TRACE_MGMT_DBG("Unpending cmnd %p (sn %u, exp_cmd_sn %u)", cmnd,
cmnd->pdu.bhs.sn, session->exp_cmd_sn);
/*
* If cmnd is pending, then the next command, if any, must be
* pending too. So, just insert a clone instead of cmnd to
* fill the hole in SNs. Then we can release cmnd.
*/
tm_clone = iscsi_create_tm_clone(cmnd);
spin_lock(&session->sn_lock);
if (tm_clone) {
TRACE_MGMT_DBG("Adding tm_clone %p after its cmnd",
tm_clone);
list_add(&tm_clone->pending_list_entry, &cmnd->pending_list_entry);
}
list_del(&cmnd->pending_list_entry);
cmnd->pending = 0;
spin_unlock(&session->sn_lock);
}
req_cmnd_release_force(cmnd);
TRACE_EXIT();
}
void iscsi_cmnd_init(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
struct iscsi_cmnd *parent)
{
atomic_set(&cmnd->ref_cnt, 1);
cmnd->scst_state = ISCSI_CMD_STATE_NEW;
cmnd->conn = conn;
cmnd->parent_req = parent;
if (!parent) {
conn_get(conn);
INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
cmnd->target_task_tag = ISCSI_RESERVED_TAG_CPU32;
spin_lock_bh(&conn->cmd_list_lock);
list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
spin_unlock_bh(&conn->cmd_list_lock);
}
}
EXPORT_SYMBOL(iscsi_cmnd_init);
static struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
struct iscsi_cmnd *parent)
{
struct iscsi_cmnd *cmnd;
/* ToDo: __GFP_NOFAIL?? */
cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL | __GFP_NOFAIL);
/* Tell Coverity about __GFP_NOFAIL. */
EXTRACHECKS_BUG_ON(!cmnd);
iscsi_cmnd_init(conn, cmnd, parent);
TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
return cmnd;
}
/* Frees a command. Also frees the additional header. */
static void cmnd_free(struct iscsi_cmnd *cmnd)
{
TRACE_ENTRY();
TRACE_DBG("cmnd %p", cmnd);
if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)))
TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, parent_req %p)",
cmnd, cmnd->scst_cmd, cmnd->scst_state, cmnd->parent_req);
/* Catch users from cmd_list or rsp_cmd_list */
EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
kfree(cmnd->pdu.ahs);
#ifdef CONFIG_SCST_EXTRACHECKS
if (unlikely(cmnd->on_write_list || cmnd->on_write_timeout_list)) {
struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
cmnd, req->opcode, req->scb[0], req->flags, req->itt,
be32_to_cpu(req->data_length), req->cmd_sn,
be32_to_cpu((__force __be32)(cmnd->pdu.datasize)));
if (unlikely(cmnd->parent_req)) {
struct iscsi_scsi_cmd_hdr *preq =
cmnd_hdr(cmnd->parent_req);
PRINT_CRIT_ERROR("%p %x %u",
preq, preq->opcode, preq->scb[0]);
}
sBUG();
}
#endif
kmem_cache_free(iscsi_cmnd_cache, cmnd);
TRACE_EXIT();
}
static void iscsi_dec_active_cmds(struct iscsi_cmnd *req)
{
struct iscsi_session *sess = req->conn->session;
TRACE_DBG("Decrementing active_cmds (req %p, sess %p, new value %d)",
req, sess, atomic_read(&sess->active_cmds) - 1);
EXTRACHECKS_BUG_ON(!req->dec_active_cmds);
atomic_dec(&sess->active_cmds);
smp_mb__after_atomic_dec();
req->dec_active_cmds = 0;
#ifdef CONFIG_SCST_EXTRACHECKS
if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
atomic_read(&sess->active_cmds));
sBUG();
}
#endif
}
/* Might be called under some lock and on SIRQ */
void cmnd_done(struct iscsi_cmnd *cmnd)
{
TRACE_ENTRY();
TRACE_DBG("cmnd %p", cmnd);
if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)))
TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, parent_req %p)",
cmnd, cmnd->scst_cmd, cmnd->scst_state,
cmnd->parent_req);
EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
EXTRACHECKS_BUG_ON(cmnd->hashed);
EXTRACHECKS_BUG_ON(cmnd->cmd_req);
EXTRACHECKS_BUG_ON(cmnd->data_out_in_data_receiving);
req_del_from_write_timeout_list(cmnd);
if (!cmnd->parent_req) {
struct iscsi_conn *conn = cmnd->conn;
struct iscsi_cmnd *rsp, *t;
TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
spin_lock_bh(&conn->cmd_list_lock);
list_del(&cmnd->cmd_list_entry);
spin_unlock_bh(&conn->cmd_list_lock);
conn_put(conn);
EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
/* Order between above and below code is important! */
if (cmnd->scst_cmd || cmnd->scst_aen) {
/*
* Tell Coverity when cmnd->scst_cmd or scst_aen is set.
*/
if (cmnd->scst_state == ISCSI_CMD_STATE_AEN)
EXTRACHECKS_BUG_ON(!cmnd->scst_aen);
else
EXTRACHECKS_BUG_ON(!cmnd->scst_cmd);
switch (cmnd->scst_state) {
case ISCSI_CMD_STATE_PROCESSED:
TRACE_DBG("cmd %p PROCESSED", cmnd);
scst_tgt_cmd_done(cmnd->scst_cmd, SCST_CONTEXT_DIRECT_ATOMIC);
break;
case ISCSI_CMD_STATE_AFTER_PREPROC:
{
/* It can be for some aborted commands */
struct scst_cmd *scst_cmd = cmnd->scst_cmd;
TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
cmnd->scst_cmd = NULL;
scst_restart_cmd(scst_cmd, SCST_PREPROCESS_STATUS_ERROR_FATAL,
SCST_CONTEXT_THREAD);
break;
}
case ISCSI_CMD_STATE_AEN:
TRACE_DBG("cmd %p AEN PROCESSED", cmnd);
scst_aen_done(cmnd->scst_aen);
break;
case ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL:
break;
default:
PRINT_CRIT_ERROR("Unexpected cmnd scst state %d",
cmnd->scst_state);
sBUG();
break;
}
}
if (cmnd->own_sg) {
TRACE_DBG("own_sg for req %p", cmnd);
if (cmnd->sg != &dummy_sg[0])
scst_free_sg(cmnd->sg, cmnd->sg_cnt);
#ifdef CONFIG_SCST_DEBUG
/*
* We can make correct behavior depend on this, so
* a bug can disappear whenever you enable debugging,
* but probability of failure on zero/NULL is much
* bigger.
*/
cmnd->own_sg = 0;
cmnd->sg = NULL;
cmnd->sg_cnt = -1;
#endif
}
if (unlikely(cmnd->dec_active_cmds))
iscsi_dec_active_cmds(cmnd);
list_for_each_entry_safe(rsp, t, &cmnd->rsp_cmd_list, rsp_cmd_list_entry)
cmnd->conn->transport->iscsit_free_cmd(rsp);
cmnd->conn->transport->iscsit_free_cmd(cmnd);
} else {
struct iscsi_cmnd *parent = cmnd->parent_req;
if (cmnd->own_sg) {
TRACE_DBG("own_sg for rsp %p", cmnd);
if (cmnd->sg != &dummy_sg[0] &&
cmnd->sg != cmnd->rsp_sg)
scst_free_sg(cmnd->sg, cmnd->sg_cnt);
#ifdef CONFIG_SCST_DEBUG
cmnd->own_sg = 0;
cmnd->sg = NULL;
cmnd->sg_cnt = -1;
#endif
}
EXTRACHECKS_BUG_ON(cmnd->dec_active_cmds);
if (cmnd == parent->main_rsp) {
TRACE_DBG("Finishing main rsp %p (req %p)",
cmnd, parent);
parent->main_rsp = NULL;
}
cmnd_put(parent);
/*
* cmnd will be freed on the last parent's put and can already
* be freed!!
*/
}
TRACE_EXIT();
}
EXPORT_SYMBOL(cmnd_done);
/*
* Corresponding conn may also get destroyed after this function, except only
* if it's called from the read thread!
*
* It can't be called in parallel with iscsi_cmnds_init_write()!
*/
void req_cmnd_release_force(struct iscsi_cmnd *req)
{
struct iscsi_cmnd *rsp, *t;
struct iscsi_conn *conn = req->conn;
LIST_HEAD(cmds_list);
TRACE_ENTRY();
if (req->force_release_done) {
/*
* There are some scenarios when this function can be called
* more, than once, for the same req. For instance, dropped
* command in iscsi_push_cmnd() and then for it
* iscsi_fail_data_waiting_cmnd() during closing (aborting) this
* connection or from iscsi_check_tm_data_wait_timeouts().
*/
TRACE_MGMT_DBG("Double force release for req %p", req);
EXTRACHECKS_BUG_ON(!req->release_called);
sBUG_ON(req->hashed);
sBUG_ON(req->cmd_req);
sBUG_ON(req->main_rsp);
sBUG_ON(!list_empty(&req->rx_ddigest_cmd_list));
sBUG_ON(req->pending);
cmnd_put(req);
goto out;
}
TRACE_MGMT_DBG("req %p", req);
req->force_release_done = 1;
sBUG_ON(req == conn->read_cmnd);
spin_lock_bh(&conn->write_list_lock);
list_for_each_entry_safe(rsp, t, &conn->write_list, write_list_entry) {
if (rsp->parent_req != req)
continue;
cmd_del_from_write_list(rsp);
list_add_tail(&rsp->write_list_entry, &cmds_list);
}
spin_unlock_bh(&conn->write_list_lock);
list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
TRACE_MGMT_DBG("Putting write rsp %p", rsp);
list_del(&rsp->write_list_entry);
cmnd_put(rsp);
}
/* Supposed nobody can add responses in the list anymore */
list_for_each_entry_reverse(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
bool r;
if (rsp->force_cleanup_done)
continue;
rsp->force_cleanup_done = 1;
if (cmnd_get_check(rsp))
continue;
spin_lock_bh(&conn->write_list_lock);
r = rsp->on_write_list || rsp->write_processing_started;
spin_unlock_bh(&conn->write_list_lock);
cmnd_put(rsp);
if (r)
continue;
/*
* If both on_write_list and write_processing_started not set,
* we can safely put() rsp.
*/
TRACE_MGMT_DBG("Putting rsp %p", rsp);
cmnd_put(rsp);
}
if (req->main_rsp) {
TRACE_MGMT_DBG("Putting main rsp %p", req->main_rsp);
cmnd_put(req->main_rsp);
req->main_rsp = NULL;
}
req_cmnd_release(req);
out:
TRACE_EXIT();
}
EXPORT_SYMBOL(req_cmnd_release_force);
void req_cmnd_pre_release(struct iscsi_cmnd *req)
{
struct iscsi_cmnd *c, *t;
TRACE_ENTRY();
TRACE_DBG("req %p", req);
#ifdef CONFIG_SCST_EXTRACHECKS
sBUG_ON(req->release_called);
req->release_called = 1;
#endif
if (unlikely(test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags))) {
TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, state %d)",
req, req->scst_cmd, req->scst_state);
}
sBUG_ON(req->parent_req);
if (unlikely(req->hashed)) {
/* It sometimes can happen during errors recovery */
TRACE_MGMT_DBG("Removing req %p from hash", req);
cmnd_remove_data_wait_hash(req);
}
if (unlikely(req->cmd_req)) {
/* It sometimes can happen during errors recovery */
TRACE_MGMT_DBG("Putting cmd_req %p (req %p)", req->cmd_req,
req);
req->cmd_req->data_out_in_data_receiving = 0;
cmnd_put(req->cmd_req);
req->cmd_req = NULL;
}
if (unlikely(req->main_rsp)) {
TRACE_DBG("Sending main rsp %p", req->main_rsp);
if (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) {
if (req->scst_cmd)
iscsi_set_resid(req->main_rsp);
else
iscsi_set_resid_no_scst_cmd(req->main_rsp);
}
iscsi_cmnd_init_write(req->main_rsp, ISCSI_INIT_WRITE_WAKE);
req->main_rsp = NULL;
}
list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list, rx_ddigest_cmd_list_entry) {
cmd_del_from_rx_ddigest_list(c);
cmnd_put(c);
}
EXTRACHECKS_BUG_ON(req->pending);
if (unlikely(req->dec_active_cmds))
iscsi_dec_active_cmds(req);
TRACE_EXIT();
}
EXPORT_SYMBOL(req_cmnd_pre_release);
/*
* Corresponding conn may also get destroyed after this function, except only
* if it's called from the read thread!
*/
static void req_cmnd_release(struct iscsi_cmnd *req)
{
TRACE_ENTRY();
req_cmnd_pre_release(req);
cmnd_put(req);
TRACE_EXIT();
}
/*
* Corresponding conn may also get destroyed after this function, except only
* if it's called from the read thread!
*/
void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
{
TRACE_DBG("%p", cmnd);
#ifdef CONFIG_SCST_EXTRACHECKS
sBUG_ON(cmnd->release_called);
cmnd->release_called = 1;
#endif
EXTRACHECKS_BUG_ON(!cmnd->parent_req);
cmnd_put(cmnd);
}
EXPORT_SYMBOL(rsp_cmnd_release);
static struct iscsi_cmnd *iscsi_alloc_rsp(struct iscsi_cmnd *parent)
{
struct iscsi_cmnd *rsp;
TRACE_ENTRY();
rsp = parent->conn->transport->iscsit_alloc_cmd(parent->conn, parent);
TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
cmnd_get(parent);
TRACE_EXIT_HRES(rsp);
return rsp;
}
static inline struct iscsi_cmnd *iscsi_alloc_main_rsp(struct iscsi_cmnd *parent)
{
struct iscsi_cmnd *rsp;
TRACE_ENTRY();
EXTRACHECKS_BUG_ON(parent->main_rsp);
rsp = iscsi_alloc_rsp(parent);
parent->main_rsp = rsp;
TRACE_EXIT_HRES(rsp);
return rsp;
}
static void iscsi_cmnds_init_write(struct list_head *send, int flags)
{
struct iscsi_cmnd *rsp = list_first_entry(send, struct iscsi_cmnd,
write_list_entry);
struct iscsi_conn *conn = rsp->conn;
struct list_head *pos, *next;
sBUG_ON(list_empty(send));
if (!(conn->ddigest_type & DIGEST_NONE)) {
list_for_each(pos, send) {
rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
if (rsp->pdu.datasize != 0) {
TRACE_DBG("Doing data digest (%p:%x)",
rsp, cmnd_opcode(rsp));
digest_tx_data(rsp);
}
}
}
spin_lock_bh(&conn->write_list_lock);
list_for_each_safe(pos, next, send) {
rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
sBUG_ON(conn != rsp->conn);
list_del(&rsp->write_list_entry);
cmd_add_on_write_list(conn, rsp);
}
spin_unlock_bh(&conn->write_list_lock);
if (flags & ISCSI_INIT_WRITE_WAKE)
conn->transport->iscsit_make_conn_wr_active(conn);
}
static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
{
LIST_HEAD(head);
#ifdef CONFIG_SCST_EXTRACHECKS
if (unlikely(rsp->on_write_list)) {
PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %u %u %d %d",
rsp->pdu.bhs.itt, cmnd_opcode(rsp), cmnd_scsicode(rsp),
rsp->hdigest, rsp->ddigest,
list_empty(&rsp->rsp_cmd_list), rsp->hashed);
sBUG();
}
#endif
list_add_tail(&rsp->write_list_entry, &head);
iscsi_cmnds_init_write(&head, flags);
}
static void iscsi_set_resid_no_scst_cmd(struct iscsi_cmnd *rsp)
{
struct iscsi_cmnd *req = rsp->parent_req;
struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
struct iscsi_scsi_rsp_hdr *rsp_hdr =
(struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
int resid, out_resid;
TRACE_ENTRY();
sBUG_ON(req->scst_cmd);
TRACE_DBG("req %p, rsp %p, outstanding_r2t %d, r2t_len_to_receive %d, r2t_len_to_send %d, not_received_data_len %d",
req, rsp, req->outstanding_r2t, req->r2t_len_to_receive,
req->r2t_len_to_send, req->not_received_data_len);
if ((req_hdr->flags & ISCSI_CMD_READ) &&
(req_hdr->flags & ISCSI_CMD_WRITE)) {
out_resid = req->not_received_data_len;
if (out_resid > 0) {
rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
rsp_hdr->residual_count = cpu_to_be32(out_resid);
} else if (out_resid < 0) {
out_resid = -out_resid;
rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
rsp_hdr->residual_count = cpu_to_be32(out_resid);
}
resid = cmnd_read_size(req);
if (resid > 0) {
rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
rsp_hdr->bi_residual_count = cpu_to_be32(resid);
} else if (resid < 0) {
resid = -resid;
rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_OVERFLOW;
rsp_hdr->bi_residual_count = cpu_to_be32(resid);
}
} else if (req_hdr->flags & ISCSI_CMD_READ) {
resid = be32_to_cpu(req_hdr->data_length);
if (resid > 0) {
rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
rsp_hdr->residual_count = cpu_to_be32(resid);
}
} else if (req_hdr->flags & ISCSI_CMD_WRITE) {
resid = req->not_received_data_len;
if (resid > 0) {
rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
rsp_hdr->residual_count = cpu_to_be32(resid);
}
}
TRACE_EXIT();
}
void iscsi_set_resid(struct iscsi_cmnd *rsp)
{
struct iscsi_cmnd *req = rsp->parent_req;
struct scst_cmd *scst_cmd = req->scst_cmd;
struct iscsi_scsi_cmd_hdr *req_hdr;
struct iscsi_scsi_rsp_hdr *rsp_hdr;
int resid, out_resid;
TRACE_ENTRY();
if (likely(!scst_get_resid(scst_cmd, &resid, &out_resid))) {
TRACE_DBG("No residuals for req %p", req);
goto out;
}
TRACE_DBG("req %p, resid %d, out_resid %d", req, resid, out_resid);
req_hdr = cmnd_hdr(req);
rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
if ((req_hdr->flags & ISCSI_CMD_READ) &&
(req_hdr->flags & ISCSI_CMD_WRITE)) {
if (out_resid > 0) {
rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
rsp_hdr->residual_count = cpu_to_be32(out_resid);
} else if (out_resid < 0) {
out_resid = -out_resid;
rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
rsp_hdr->residual_count = cpu_to_be32(out_resid);
}
if (resid > 0) {
rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
rsp_hdr->bi_residual_count = cpu_to_be32(resid);
} else if (resid < 0) {
resid = -resid;
rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_OVERFLOW;
rsp_hdr->bi_residual_count = cpu_to_be32(resid);
}
} else {
if (resid > 0) {
rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
rsp_hdr->residual_count = cpu_to_be32(resid);
} else if (resid < 0) {
resid = -resid;
rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
rsp_hdr->residual_count = cpu_to_be32(resid);
}
}
out:
TRACE_EXIT();
}
EXPORT_SYMBOL(iscsi_set_resid);
static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
{
struct iscsi_cmnd *rsp;
struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
struct iscsi_data_in_hdr *rsp_hdr;
u32 pdusize, size, offset, sn;
LIST_HEAD(send);
TRACE_DBG("req %p", req);
pdusize = req->conn->session->sess_params.max_xmit_data_length;
size = req->bufflen;
offset = 0;
sn = 0;
while (1) {
rsp = iscsi_alloc_rsp(req);
TRACE_DBG("rsp %p", rsp);
rsp->sg = req->sg;
rsp->sg_cnt = req->sg_cnt;
rsp->bufflen = req->bufflen;
rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
rsp_hdr->itt = req_hdr->itt;
rsp_hdr->ttt = ISCSI_RESERVED_TAG;
rsp_hdr->buffer_offset = cpu_to_be32(offset);
rsp_hdr->data_sn = cpu_to_be32(sn);
if (size <= pdusize) {
TRACE_DBG("offset %d, size %d", offset, size);
rsp->pdu.datasize = size;
if (send_status) {
TRACE_DBG("status %x", status);
rsp_hdr->flags = ISCSI_FLG_FINAL |
ISCSI_FLG_STATUS;
rsp_hdr->cmd_status = status;
iscsi_set_resid(rsp);
}
list_add_tail(&rsp->write_list_entry, &send);
break;
}
TRACE_DBG("pdusize %d, offset %d, size %d",
pdusize, offset, size);
rsp->pdu.datasize = pdusize;
size -= pdusize;
offset += pdusize;
sn++;
list_add_tail(&rsp->write_list_entry, &send);
}
iscsi_cmnds_init_write(&send, 0);
}
static void iscsi_tcp_set_sense_data(struct iscsi_cmnd *rsp, const u8 *sense_buf, int sense_len)
{
struct scatterlist *sg;
rsp->sg = rsp->rsp_sg;
rsp->sg_cnt = 2;
rsp->own_sg = 1;
sg = rsp->sg;
sg_init_table(sg, 2);
sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
sg_set_buf(&sg[1], (u8 *)sense_buf, sense_len);
}
static void iscsi_init_status_rsp(struct iscsi_cmnd *rsp, int status, const u8 *sense_buf,
int sense_len)
{
struct iscsi_cmnd *req = rsp->parent_req;
struct iscsi_scsi_rsp_hdr *rsp_hdr;
TRACE_ENTRY();
rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
rsp_hdr->flags = ISCSI_FLG_FINAL;
rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
rsp_hdr->cmd_status = status;
rsp_hdr->itt = cmnd_hdr(req)->itt;
if (scst_sense_valid(sense_buf)) {
TRACE_DBG("SENSE VALID");
rsp->sense_hdr.length = cpu_to_be16(sense_len);
rsp->conn->transport->iscsit_set_sense_data(rsp, sense_buf,
sense_len);
rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
rsp->bufflen = rsp->pdu.datasize;
} else {
rsp->pdu.datasize = 0;
rsp->bufflen = 0;
}
TRACE_EXIT();
}
struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status, const u8 *sense_buf,
int sense_len)
{
struct iscsi_cmnd *rsp;
TRACE_ENTRY();
rsp = iscsi_alloc_rsp(req);
TRACE_DBG("rsp %p", rsp);
iscsi_init_status_rsp(rsp, status, sense_buf, sense_len);
iscsi_set_resid(rsp);
TRACE_EXIT_HRES(rsp);
return rsp;
}
EXPORT_SYMBOL(create_status_rsp);
static void iscsi_tcp_send_data_rsp(struct iscsi_cmnd *req, u8 *sense, int sense_len, u8 status,
int is_send_status)
{
if (status != SAM_STAT_CHECK_CONDITION &&
(cmnd_hdr(req)->flags & (ISCSI_CMD_WRITE | ISCSI_CMD_READ)) !=
(ISCSI_CMD_WRITE | ISCSI_CMD_READ)) {
send_data_rsp(req, status, is_send_status);
} else {
struct iscsi_cmnd *rsp;
send_data_rsp(req, 0, 0);
if (is_send_status) {
rsp = create_status_rsp(req, status, sense, sense_len);
iscsi_cmnd_init_write(rsp, 0);
}
}
}
/*
* Initializes data receive fields. Can be called only when they have not been
* initialized yet.
*/
static int iscsi_set_prelim_r2t_len_to_receive(struct iscsi_cmnd *req)
{
struct iscsi_scsi_cmd_hdr *req_hdr =
(struct iscsi_scsi_cmd_hdr *)&req->pdu.bhs;
int res = 0;
unsigned int not_received;
TRACE_ENTRY();
if (req_hdr->flags & ISCSI_CMD_FINAL) {
if (req_hdr->flags & ISCSI_CMD_WRITE)
iscsi_set_not_received_data_len(req,
be32_to_cpu(req_hdr->data_length) - req->pdu.datasize);
goto out;
}
sBUG_ON(req->outstanding_r2t != 0);
res = cmnd_insert_data_wait_hash(req);
if (res != 0) {
/*
* We have to close connection, because otherwise a data
* corruption is possible if we allow to receive data
* for this request in another request with duplicated ITT.
*/
mark_conn_closed(req->conn);
goto out;
}
/*
* We need to wait for one or more PDUs. Let's simplify
* other code and pretend we need to receive 1 byte.
* In data_out_start() we will correct it.
*/
req->outstanding_r2t = 1;
req_add_to_write_timeout_list(req);
req->r2t_len_to_receive = 1;
req->r2t_len_to_send = 0;
not_received = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
not_received -= min_t(unsigned int, not_received,
req->conn->session->sess_params.first_burst_length);
iscsi_set_not_received_data_len(req, not_received);
TRACE_DBG("req %p, op %x, outstanding_r2t %d, r2t_len_to_receive %d, r2t_len_to_send %d, not_received_data_len %d",
req,
cmnd_opcode(req), req->outstanding_r2t, req->r2t_len_to_receive,
req->r2t_len_to_send, req->not_received_data_len);
out:
TRACE_EXIT_RES(res);
return res;
}
static int create_preliminary_no_scst_rsp(struct iscsi_cmnd *req, int status, const u8 *sense_buf,
int sense_len)
{
struct iscsi_cmnd *rsp;
int res = 0;
TRACE_ENTRY();
if (req->prelim_compl_flags != 0) {
TRACE_MGMT_DBG("req %p already prelim completed", req);
goto out;
}
req->scst_state = ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL;
sBUG_ON(req->scst_cmd);
res = iscsi_preliminary_complete(req, req, true);
rsp = iscsi_alloc_main_rsp(req);
TRACE_DBG("main rsp %p", rsp);
iscsi_init_status_rsp(rsp, status, sense_buf, sense_len);
/* Resid will be set in req_cmnd_release() */
out:
TRACE_EXIT_RES(res);
return res;
}
int set_scst_preliminary_status_rsp(struct iscsi_cmnd *req, bool get_data, int key, int asc,
int ascq)
{
int res = 0;
TRACE_ENTRY();
if (!req->scst_cmd) {
/* There must be already error set */
goto complete;
}
scst_set_cmd_error(req->scst_cmd, key, asc, ascq);
complete:
res = iscsi_preliminary_complete(req, req, get_data);
TRACE_EXIT_RES(res);
return res;
}
static int create_reject_rsp(struct iscsi_cmnd *req, int reason, bool get_data)
{
int res = 0;
struct iscsi_cmnd *rsp;
struct iscsi_reject_hdr *rsp_hdr;
struct scatterlist *sg;
TRACE_ENTRY();
TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
if (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) {
if (!req->scst_cmd) {
/* BUSY status must be already set */
struct iscsi_scsi_rsp_hdr *rsp_hdr1;
rsp_hdr1 = (struct iscsi_scsi_rsp_hdr *)
&req->main_rsp->pdu.bhs;
sBUG_ON(rsp_hdr1->cmd_status == 0);
/*
* Let's not send REJECT here. The initiator will retry
* and, hopefully, next time we will not fail allocating
* scst_cmd, so we will then send the REJECT.
*/
goto out;
} else {
/*
* "In all the cases in which a pre-instantiated SCSI
* task is terminated because of the reject, the target
* MUST issue a proper SCSI command response with CHECK
* CONDITION as described in Section 10.4.3 Response" -
* RFC 3720.
*/
set_scst_preliminary_status_rsp(req, get_data,
SCST_LOAD_SENSE(scst_sense_invalid_message));
}
}
rsp = iscsi_alloc_main_rsp(req);
rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_REJECT;
rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
rsp_hdr->reason = reason;
rsp->sg = rsp->rsp_sg;
rsp->sg_cnt = 1;
rsp->own_sg = 1;
sg = rsp->sg;
sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
rsp->pdu.datasize = sizeof(struct iscsi_hdr);
rsp->bufflen = rsp->pdu.datasize;
res = iscsi_preliminary_complete(req, req, true);
out:
TRACE_EXIT_RES(res);
return res;
}
static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
{
int res = max(-1,
(int)sess->tgt_params.queued_cmnds - atomic_read(&sess->active_cmds) - 1);
TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
sess, atomic_read(&sess->active_cmds));
return res;
}
__be32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
{
struct iscsi_conn *conn = cmnd->conn;
struct iscsi_session *sess = conn->session;
__be32 res;
spin_lock(&sess->sn_lock);
if (set_stat_sn)
cmnd->pdu.bhs.sn = (__force u32)cpu_to_be32(conn->stat_sn++);
cmnd->pdu.bhs.exp_sn = (__force u32)cpu_to_be32(sess->exp_cmd_sn);
cmnd->pdu.bhs.max_sn = (__force u32)cpu_to_be32(sess->exp_cmd_sn +
iscsi_get_allowed_cmds(sess));
res = cpu_to_be32(conn->stat_sn);
spin_unlock(&sess->sn_lock);
return res;
}
EXPORT_SYMBOL(cmnd_set_sn);
/* Called under sn_lock */
static void update_stat_sn(struct iscsi_cmnd *cmnd)
{
struct iscsi_conn *conn = cmnd->conn;
u32 exp_stat_sn;
exp_stat_sn = be32_to_cpu((__force __be32)cmnd->pdu.bhs.exp_sn);
cmnd->pdu.bhs.exp_sn = exp_stat_sn;
TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
(int)(exp_stat_sn - conn->stat_sn) <= 0) {
/* free pdu resources */
cmnd->conn->exp_stat_sn = exp_stat_sn;
}
}
static struct iscsi_cmnd *cmnd_find_itt_get(struct iscsi_conn *conn, __be32 itt)
{
struct iscsi_cmnd *cmnd, *found_cmnd = NULL;
spin_lock_bh(&conn->cmd_list_lock);
list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
if (cmnd->pdu.bhs.itt == itt && !cmnd_get_check(cmnd)) {
found_cmnd = cmnd;
break;
}
}
spin_unlock_bh(&conn->cmd_list_lock);
return found_cmnd;
}
/*
** We use the ITT hash only to find original request PDU for subsequent
** Data-Out PDUs.
**/
/* Must be called under cmnd_data_wait_hash_lock */
static struct iscsi_cmnd *__cmnd_find_data_wait_hash(struct iscsi_conn *conn, __be32 itt)
{
struct list_head *head;
struct iscsi_cmnd *cmnd;
head = &conn->session->cmnd_data_wait_hash[cmnd_hashfn((__force u32)itt)];
list_for_each_entry(cmnd, head, hash_list_entry) {
if (cmnd->pdu.bhs.itt == itt)
return cmnd;
}
return NULL;
}
static struct iscsi_cmnd *cmnd_find_data_wait_hash(struct iscsi_conn *conn, __be32 itt)
{
struct iscsi_cmnd *res;
struct iscsi_session *session = conn->session;
spin_lock(&session->cmnd_data_wait_hash_lock);
res = __cmnd_find_data_wait_hash(conn, itt);
spin_unlock(&session->cmnd_data_wait_hash_lock);
return res;
}
static inline u32 get_next_ttt(struct iscsi_conn *conn)
{
u32 ttt;
struct iscsi_session *session = conn->session;
/* Not compatible with MC/S! */
iscsi_extracheck_is_rd_thread(conn);
if (unlikely(session->next_ttt == ISCSI_RESERVED_TAG_CPU32))
session->next_ttt++;
ttt = session->next_ttt++;
return ttt;
}
static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd)
{
struct iscsi_session *session = cmnd->conn->session;
struct iscsi_cmnd *tmp;
struct list_head *head;
int err = 0;
__be32 itt = cmnd->pdu.bhs.itt;
if (unlikely(cmnd->hashed)) {
/*
* It can be for preliminary completed commands, when this
* function already failed.
*/
goto out;
}
/*
* We don't need TTT, because ITT/buffer_offset pair is sufficient
* to find out the original request and buffer for Data-Out PDUs, but
* crazy iSCSI spec requires us to send this superfluous field in
* R2T PDUs and some initiators may rely on it.
*/
cmnd->target_task_tag = get_next_ttt(cmnd->conn);
TRACE_DBG("%p:%x", cmnd, itt);
if (unlikely(itt == ISCSI_RESERVED_TAG)) {
PRINT_ERROR("ITT is RESERVED_TAG (conn %p)", cmnd->conn);
PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs, sizeof(cmnd->pdu.bhs));
err = -ISCSI_REASON_PROTOCOL_ERROR;
goto out;
}
spin_lock(&session->cmnd_data_wait_hash_lock);
head = &session->cmnd_data_wait_hash[cmnd_hashfn((__force u32)itt)];
tmp = __cmnd_find_data_wait_hash(cmnd->conn, itt);
if (likely(!tmp)) {
TRACE_DBG("Adding cmnd %p to the hash (ITT %x)",
cmnd, cmnd->pdu.bhs.itt);
list_add_tail(&cmnd->hash_list_entry, head);
cmnd->hashed = 1;
} else {
PRINT_ERROR("Task %x in progress, cmnd %p (conn %p)",
itt, cmnd, cmnd->conn);
err = -ISCSI_REASON_TASK_IN_PROGRESS;
}
spin_unlock(&session->cmnd_data_wait_hash_lock);
out:
return err;
}
static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd)
{
struct iscsi_session *session = cmnd->conn->session;
struct iscsi_cmnd *tmp;
spin_lock(&session->cmnd_data_wait_hash_lock);
tmp = __cmnd_find_data_wait_hash(cmnd->conn, cmnd->pdu.bhs.itt);
if (likely(tmp && tmp == cmnd)) {
TRACE_DBG("Deleting cmnd %p from the hash (ITT %x)", cmnd,
cmnd->pdu.bhs.itt);
list_del(&cmnd->hash_list_entry);
cmnd->hashed = 0;
} else {
PRINT_ERROR("%p:%x not found", cmnd, cmnd->pdu.bhs.itt);
}
spin_unlock(&session->cmnd_data_wait_hash_lock);
}
static void cmnd_prepare_get_rejected_immed_data(struct iscsi_cmnd *cmnd)
{
struct iscsi_conn *conn = cmnd->conn;
struct scatterlist *sg = cmnd->sg;
char *addr;
u32 size, s, e;
unsigned int i;
TRACE_ENTRY();
TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(cmnd),
"Skipping (cmnd %p, ITT %x, op %x, cmd op %x, datasize %u, scst_cmd %p, scst state %d, status %d)",
cmnd,
cmnd->pdu.bhs.itt, cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state,
cmnd->scst_cmd ? cmnd->scst_cmd->status : -1);
iscsi_extracheck_is_rd_thread(conn);
size = cmnd->pdu.datasize;
if (!size)
goto out;
/* We already checked pdu.datasize in check_segment_length() */
/*
* There are no problems with the safety from concurrent
* accesses to dummy_page in dummy_sg, since data only
* will be read and then discarded.
*/
sg = &dummy_sg[0];
if (!cmnd->sg) {
/* just in case */
cmnd->sg = sg;
cmnd->bufflen = PAGE_SIZE;
cmnd->own_sg = 1;
}
addr = page_address(sg_page(&sg[0]));
for (s = size, i = 0; s > 0; i++, s -= e) {
/* We already checked pdu.datasize in check_segment_length() */
sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
conn->read_iov[i].iov_base = addr;
e = min_t(u32, s, PAGE_SIZE);
conn->read_iov[i].iov_len = e;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)
iov_iter_kvec(&conn->read_msg.msg_iter, READ, conn->read_iov, i, size);
#else
conn->read_msg.msg_iov = conn->read_iov;
conn->read_msg.msg_iovlen = i;
conn->read_size = size;
#endif
out:
TRACE_EXIT();
}
int iscsi_preliminary_complete(struct iscsi_cmnd *req, struct iscsi_cmnd *orig_req, bool get_data)
{
int res = 0;
bool set_r2t_len;
struct iscsi_hdr *orig_req_hdr = &orig_req->pdu.bhs;
TRACE_ENTRY();
#ifdef CONFIG_SCST_DEBUG
{
struct iscsi_hdr *req_hdr = &req->pdu.bhs;
TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(orig_req),
"Prelim completed req %p, orig_req %p (FINAL %x, outstanding_r2t %d)",
req, orig_req, (req_hdr->flags & ISCSI_CMD_FINAL),
orig_req->outstanding_r2t);
}
#endif
iscsi_extracheck_is_rd_thread(req->conn);
sBUG_ON(req->parent_req);
if (test_bit(ISCSI_CMD_PRELIM_COMPLETED, &req->prelim_compl_flags)) {
TRACE_MGMT_DBG("req %p already prelim completed", req);
/* To not try to get data twice */
get_data = false;
}
/*
* We need to receive all outstanding PDUs, even if direction isn't
* WRITE. Test of PRELIM_COMPLETED is needed, because
* iscsi_set_prelim_r2t_len_to_receive() could also have failed before.
*/
set_r2t_len = !orig_req->hashed &&
(cmnd_opcode(orig_req) == ISCSI_OP_SCSI_CMD) &&
!test_bit(ISCSI_CMD_PRELIM_COMPLETED,
&orig_req->prelim_compl_flags);
TRACE_DBG("get_data %d, set_r2t_len %d", get_data, set_r2t_len);
if (get_data)
cmnd_prepare_get_rejected_immed_data(req);
if (test_bit(ISCSI_CMD_PRELIM_COMPLETED, &orig_req->prelim_compl_flags))
goto out_set;
if (set_r2t_len) {
res = iscsi_set_prelim_r2t_len_to_receive(orig_req);
} else if (orig_req_hdr->flags & ISCSI_CMD_WRITE) {
/*
* We will get here if orig_req prelim completed in the middle
* of data receiving. We won't send more R2T's, so
* r2t_len_to_send is final and won't be updated anymore in
* future.
*/
iscsi_set_not_received_data_len(orig_req, orig_req->r2t_len_to_send);
}
out_set:
set_bit(ISCSI_CMD_PRELIM_COMPLETED, &orig_req->prelim_compl_flags);
set_bit(ISCSI_CMD_PRELIM_COMPLETED, &req->prelim_compl_flags);
TRACE_EXIT_RES(res);
return res;
}
static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmd, u32 offset,
u32 size)
{
struct scatterlist *sg = cmd->sg;
unsigned int bufflen = cmd->bufflen;
unsigned int idx, i, buff_offs;
const u32 read_size = size;
int res = 0;
TRACE_ENTRY();
TRACE_DBG("cmd %p, sg %p, offset %u, size %u",
cmd, cmd->sg, offset, size);
iscsi_extracheck_is_rd_thread(conn);
buff_offs = offset;
offset += sg[0].offset;
idx = offset >> PAGE_SHIFT;
offset &= ~PAGE_MASK;
i = 0;
while (1) {
unsigned int sg_len;
char *addr;
if (unlikely(buff_offs >= bufflen)) {
TRACE_DBG("Residual overflow (cmd %p, buff_offs %d, bufflen %d)",
cmd, buff_offs, bufflen);
idx = 0;
sg = &dummy_sg[0];
offset = 0;
}
/* To suppress a false positive Coverity complaint. */
EXTRACHECKS_BUG_ON(sg == &dummy_sg[0] && idx > 0);
addr = page_address(sg_page(&sg[idx]));
EXTRACHECKS_BUG_ON(!addr);
sg_len = sg[idx].offset + sg[idx].length - offset;
conn->read_iov[i].iov_base = addr + offset;
if (size <= sg_len) {
TRACE_DBG("idx=%d, i=%d, offset=%u, size=%d, addr=%p",
idx, i, offset, size, addr);
conn->read_iov[i].iov_len = size;
break;
}
conn->read_iov[i].iov_len = sg_len;
TRACE_DBG("idx=%d, i=%d, offset=%u, size=%d, sg_len=%u, addr=%p",
idx, i, offset, size, sg_len, addr);
size -= sg_len;
buff_offs += sg_len;
i++;
if (unlikely(i >= ISCSI_CONN_IOV_MAX)) {
PRINT_ERROR("Initiator %s violated negotiated parameters by sending too much data (size left %d), conn %p",
conn->session->initiator_name, size, conn);
mark_conn_closed(conn);
res = -EINVAL;
goto out;
}
idx++;
offset = 0;
}
i++;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)
iov_iter_kvec(&conn->read_msg.msg_iter, READ, conn->read_iov, i,
read_size);
#else
conn->read_msg.msg_iov = conn->read_iov;
conn->read_msg.msg_iovlen = i;
conn->read_size = read_size;
#endif
TRACE_DBG("msg_iov=%p, msg_iovlen=%u", conn->read_iov, i);
out:
TRACE_EXIT_RES(res);
return res;
}
static void send_r2t(struct iscsi_cmnd *req)
{
struct iscsi_session *sess = req->conn->session;
struct iscsi_cmnd *rsp;
struct iscsi_r2t_hdr *rsp_hdr;
u32 offset, burst;
LIST_HEAD(send);
TRACE_ENTRY();
EXTRACHECKS_BUG_ON(req->r2t_len_to_send == 0);
/*
* There is no race with data_out_start() and conn_abort(), since
* all functions called from single read thread
*/
iscsi_extracheck_is_rd_thread(req->conn);
/*
* We don't need to check for PRELIM_COMPLETED here, because for such
* commands we set r2t_len_to_send = 0, hence made sure we won't be
* called here.
*/
EXTRACHECKS_BUG_ON(req->outstanding_r2t >
sess->sess_params.max_outstanding_r2t);
if (req->outstanding_r2t == sess->sess_params.max_outstanding_r2t)
goto out;
burst = sess->sess_params.max_burst_length;
offset = be32_to_cpu(cmnd_hdr(req)->data_length) -
req->r2t_len_to_send;
do {
rsp = iscsi_alloc_rsp(req);
rsp->pdu.bhs.ttt = (__force __be32)req->target_task_tag;
rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_R2T;
rsp_hdr->flags = ISCSI_FLG_FINAL;
rsp_hdr->lun = cmnd_hdr(req)->lun;
rsp_hdr->itt = cmnd_hdr(req)->itt;
rsp_hdr->r2t_sn = (__force u32)cpu_to_be32(req->r2t_sn++);
rsp_hdr->buffer_offset = cpu_to_be32(offset);
if (req->r2t_len_to_send > burst) {
rsp_hdr->data_length = cpu_to_be32(burst);
req->r2t_len_to_send -= burst;
offset += burst;
} else {
rsp_hdr->data_length =
cpu_to_be32(req->r2t_len_to_send);
req->r2t_len_to_send = 0;
}
TRACE_WRITE("req %p, data_length %u, buffer_offset %u, r2t_sn %u, outstanding_r2t %u",
req, be32_to_cpu(rsp_hdr->data_length),
be32_to_cpu(rsp_hdr->buffer_offset),
be32_to_cpu((__force __be32)rsp_hdr->r2t_sn),
req->outstanding_r2t);
list_add_tail(&rsp->write_list_entry, &send);
req->outstanding_r2t++;
} while (req->outstanding_r2t < sess->sess_params.max_outstanding_r2t &&
req->r2t_len_to_send != 0);
iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
out:
TRACE_EXIT();
}
static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
{
int res = SCST_PREPROCESS_STATUS_SUCCESS;
struct iscsi_cmnd *req = scst_cmd_get_tgt_priv(scst_cmd);
struct iscsi_cmnd *c, *t;
TRACE_ENTRY();
EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
/* If data digest isn't used this list will be empty */
list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list, rx_ddigest_cmd_list_entry) {
TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
if (digest_rx_data(c) != 0) {
scst_set_cmd_error(scst_cmd, SCST_LOAD_SENSE(iscsi_sense_crc_error));
res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
/*
* The rest of rx_ddigest_cmd_list will be freed
* in req_cmnd_release()
*/
goto out;
}
cmd_del_from_rx_ddigest_list(c);
cmnd_put(c);
}
out:
TRACE_EXIT_RES(res);
return res;
}
static int nop_out_start(struct iscsi_cmnd *cmnd)
{
struct iscsi_conn *conn = cmnd->conn;
struct iscsi_hdr *req_hdr = &cmnd->pdu.bhs;
u32 size, tmp;
int i, err = 0;
TRACE_DBG("%p", cmnd);
iscsi_extracheck_is_rd_thread(conn);
if (!(req_hdr->flags & ISCSI_FLG_FINAL)) {
PRINT_ERROR("Initiator sent Nop-Out with not a single PDU");
err = -ISCSI_REASON_PROTOCOL_ERROR;
goto out;
}
if (cmnd->pdu.bhs.itt == ISCSI_RESERVED_TAG) {
if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
PRINT_ERROR("Initiator sent RESERVED tag for non-immediate Nop-Out command");
}
update_stat_sn(cmnd);
size = cmnd->pdu.datasize;
if (size && !conn->session->sess_params.rdma_extensions) {
if (cmnd->pdu.bhs.itt != ISCSI_RESERVED_TAG) {
struct scatterlist *sg;
sg = scst_alloc_sg(size, GFP_KERNEL, &cmnd->sg_cnt);
if (!sg) {
TRACE(TRACE_OUT_OF_MEM, "Allocation of buffer for %d Nop-Out payload failed",
size);
err = -ISCSI_REASON_OUT_OF_RESOURCES;
goto out;
}
cmnd->sg = sg;
/* We already checked it in check_segment_length() */
sBUG_ON(cmnd->sg_cnt > (signed int)ISCSI_CONN_IOV_MAX);
cmnd->own_sg = 1;
cmnd->bufflen = size;
for (i = 0; i < cmnd->sg_cnt; i++) {
conn->read_iov[i].iov_base = page_address(sg_page(&sg[i]));
tmp = min_t(u32, size, PAGE_SIZE);
conn->read_iov[i].iov_len = tmp;
size -= tmp;
}
sBUG_ON(size != 0);
} else {
/*
* There are no problems with the safety from concurrent
* accesses to dummy_page, since for ISCSI_RESERVED_TAG
* the data only read and then discarded.
*/
for (i = 0; i < (signed int)ISCSI_CONN_IOV_MAX; i++) {
conn->read_iov[i].iov_base =
page_address(dummy_page);
tmp = min_t(u32, size, PAGE_SIZE);
conn->read_iov[i].iov_len = tmp;
size -= tmp;
}
/* We already checked size in check_segment_length() */
sBUG_ON(size != 0);
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)
iov_iter_kvec(&conn->read_msg.msg_iter, READ, conn->read_iov,
i, cmnd->pdu.datasize);
#else
conn->read_msg.msg_iov = conn->read_iov;
conn->read_msg.msg_iovlen = i;
conn->read_size = cmnd->pdu.datasize;
#endif
TRACE_DBG("msg_iov=%p, msg_iovlen=%d", conn->read_iov, i);
}
out:
return err;
}
int iscsi_cmnd_set_write_buf(struct iscsi_cmnd *req)
{
struct iscsi_conn *conn = req->conn;
struct iscsi_session *session = conn->session;
struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
struct scst_cmd *scst_cmd = req->scst_cmd;
bool unsolicited_data_expected = false;
int res = 0;
req->bufflen = scst_cmd_get_write_fields(scst_cmd, &req->sg, &req->sg_cnt);
unsolicited_data_expected = !(req_hdr->flags & ISCSI_CMD_FINAL);
if (unlikely(session->sess_params.initial_r2t && unsolicited_data_expected)) {
PRINT_ERROR("Initiator %s violated negotiated parameters: initial R2T is required (ITT %x, op %x, conn %p)",
session->initiator_name, req->pdu.bhs.itt,
req_hdr->scb[0], conn);
res = -EINVAL;
goto out_close;
}
if (unlikely(!session->sess_params.immediate_data && req->pdu.datasize)) {
PRINT_ERROR("Initiator %s violated negotiated parameters: forbidden immediate data sent (ITT %x, op %x, conn %p)",
session->initiator_name,
req->pdu.bhs.itt, req_hdr->scb[0], conn);
res = -EINVAL;
goto out_close;
}
if (unlikely(session->sess_params.first_burst_length < req->pdu.datasize)) {
PRINT_ERROR("Initiator %s violated negotiated parameters: immediate data len (%d) > first_burst_length (%d) (ITT %x, op %x, conn %p)",
session->initiator_name,
req->pdu.datasize,
session->sess_params.first_burst_length,
req->pdu.bhs.itt, req_hdr->scb[0], conn);
res = -EINVAL;
goto out_close;
}
req->r2t_len_to_receive = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
/*
* In case of residual overflow req->r2t_len_to_receive and
* req->pdu.datasize might be > req->bufflen
*/
res = cmnd_insert_data_wait_hash(req);
if (unsolicited_data_expected) {
req->outstanding_r2t = 1;
req->r2t_len_to_send = req->r2t_len_to_receive -
min_t(unsigned int,
session->sess_params.first_burst_length - req->pdu.datasize,
req->r2t_len_to_receive);
} else {
req->r2t_len_to_send = req->r2t_len_to_receive;
}
if (likely(res == 0))
req_add_to_write_timeout_list(req);
out_close:
return res;
}
EXPORT_SYMBOL(iscsi_cmnd_set_write_buf);
int cmnd_rx_continue(struct iscsi_cmnd *req)
{
struct iscsi_conn *conn = req->conn;
struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
struct scst_cmd *scst_cmd = req->scst_cmd;
scst_data_direction dir;
int res = 0;
TRACE_ENTRY();
TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC);
dir = scst_cmd_get_data_direction(scst_cmd);
/*
* Check for preliminary completion here to save R2Ts. For TASK QUEUE
* FULL statuses that might be a big performance win.
*/
if (unlikely(scst_cmd_prelim_completed(scst_cmd) || req->prelim_compl_flags != 0)) {
/*
* If necessary, ISCSI_CMD_ABORTED will be set by
* iscsi_xmit_response().
*/
res = iscsi_preliminary_complete(req, req, true);
goto trace;
}
/* For prelim completed commands sg & K can be already set! */
if (dir & SCST_DATA_WRITE) {
res = iscsi_cmnd_set_write_buf(req);
if (unlikely(res != 0)) {
/*
* We have to close connection, because otherwise a data
* corruption is possible if we allow to receive data
* for this request in another request with duplicated
* ITT.
*/
goto out_close;
}
if (req->pdu.datasize) {
res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
/* For performance better to send R2Ts ASAP */
if (likely(res == 0) && req->r2t_len_to_send != 0)
send_r2t(req);
}
} else {
req->sg = scst_cmd_get_sg(scst_cmd);
req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
req->bufflen = scst_cmd_get_bufflen(scst_cmd);
if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) || req->pdu.datasize)) {
PRINT_ERROR("Unexpected unsolicited data (ITT %x CDB %x)",
req->pdu.bhs.itt, req_hdr->scb[0]);
set_scst_preliminary_status_rsp(req, true,
SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
}
}
trace:
TRACE_DBG("req=%p, dir=%d, r2t_len_to_receive=%d, r2t_len_to_send=%d, bufflen=%d, own_sg %d",
req, dir, req->r2t_len_to_receive,
req->r2t_len_to_send, req->bufflen, req->own_sg);
out:
TRACE_EXIT_RES(res);
return res;
out_close:
mark_conn_closed(conn);
res = -EINVAL;
goto out;
}
static int scsi_cmnd_start(struct iscsi_cmnd *req)
{
struct iscsi_conn *conn = req->conn;
struct iscsi_session *session = conn->session;
struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
struct scst_cmd *scst_cmd;
scst_data_direction dir;
struct iscsi_ahs_hdr *ahdr;
int res = 0;
TRACE_ENTRY();
TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, new value %d)",
req, session, atomic_read(&session->active_cmds) + 1);
atomic_inc(&session->active_cmds);
req->dec_active_cmds = 1;
EXTRACHECKS_BUG_ON(!session->scst_sess);
scst_cmd = scst_rx_cmd(session->scst_sess, (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
if (!scst_cmd) {
res = create_preliminary_no_scst_rsp(req, SAM_STAT_BUSY, NULL, 0);
goto out;
}
req->scst_cmd = scst_cmd;
scst_cmd_set_tag(scst_cmd, (__force u32)req_hdr->itt);
scst_cmd_set_tgt_priv(scst_cmd, req);
if ((req_hdr->flags & ISCSI_CMD_READ) &&
(req_hdr->flags & ISCSI_CMD_WRITE)) {
int sz = cmnd_read_size(req);
if (unlikely(sz < 0)) {
PRINT_ERROR("BIDI data transfer, but initiator not supplied Bidirectional Read Expected Data Transfer Length AHS");
set_scst_preliminary_status_rsp(req, true,
SCST_LOAD_SENSE(scst_sense_parameter_value_invalid));
} else {
dir = SCST_DATA_BIDI;
scst_cmd_set_expected(scst_cmd, dir, sz);
scst_cmd_set_expected_out_transfer_len(scst_cmd,
be32_to_cpu(req_hdr->data_length));
if (conn->transport->need_alloc_write_buf)
scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
}
} else if (req_hdr->flags & ISCSI_CMD_READ) {
dir = SCST_DATA_READ;
scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
if (conn->transport->need_alloc_write_buf)
scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
} else if (req_hdr->flags & ISCSI_CMD_WRITE) {
dir = SCST_DATA_WRITE;
scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
} else {
dir = SCST_DATA_NONE;
scst_cmd_set_expected(scst_cmd, dir, 0);
}
switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
case ISCSI_CMD_SIMPLE:
scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_SIMPLE);
break;
case ISCSI_CMD_HEAD_OF_QUEUE:
scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_HEAD_OF_QUEUE);
break;
case ISCSI_CMD_ORDERED:
scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_ORDERED);
break;
case ISCSI_CMD_ACA:
scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_ACA);
break;
case ISCSI_CMD_UNTAGGED:
scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_UNTAGGED);
break;
default:
PRINT_WARNING("Unknown task code %x, use ORDERED instead",
req_hdr->flags & ISCSI_CMD_ATTR_MASK);
scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_ORDERED);
break;
}
scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
if (ahdr) {
uint8_t *p = (uint8_t *)ahdr;
unsigned int size = 0;
do {
int s;
ahdr = (struct iscsi_ahs_hdr *)p;
if (ahdr->ahstype == ISCSI_AHSTYPE_CDB) {
struct iscsi_cdb_ahdr *eca = (struct iscsi_cdb_ahdr *)ahdr;
scst_cmd_set_ext_cdb(scst_cmd, eca->cdb,
be16_to_cpu(ahdr->ahslength) - 1, GFP_KERNEL);
break;
}
s = 3 + be16_to_cpu(ahdr->ahslength);
s = (s + 3) & -4;
size += s;
p += s;
} while (size < req->pdu.ahssize);
}
TRACE_DBG("START Command (itt %x, queue_type %d)",
req_hdr->itt, scst_cmd_get_queue_type(scst_cmd));
req->scst_state = ISCSI_CMD_STATE_RX_CMD;
conn->rx_task = current;
scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
#ifdef CONFIG_SCST_EXTRACHECKS
conn->rx_task = NULL;
#endif
if (req->scst_state != ISCSI_CMD_STATE_RX_CMD) {
res = req->conn->transport->iscsit_receive_cmnd_data(req);
} else {
TRACE_DBG("Delaying req %p post processing (scst_state %d)",
req, req->scst_state);
res = 1;
}
out:
TRACE_EXIT_RES(res);
return res;
}
static int data_out_start(struct iscsi_cmnd *cmnd)
{
struct iscsi_conn *conn = cmnd->conn;
struct iscsi_data_out_hdr *req_hdr =
(struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
struct iscsi_cmnd *orig_req;
#if 0
struct iscsi_hdr *orig_req_hdr;
#endif
u32 offset = be32_to_cpu(req_hdr->buffer_offset);
int res = 0;
TRACE_ENTRY();
/*
* There is no race with send_r2t(), conn_abort() and
* iscsi_check_tm_data_wait_timeouts(), since
* all the functions called from single read thread
*/
iscsi_extracheck_is_rd_thread(cmnd->conn);
update_stat_sn(cmnd);
orig_req = cmnd_find_data_wait_hash(conn, req_hdr->itt);
cmnd->cmd_req = orig_req;
if (unlikely(!orig_req)) {
/*
* It shouldn't happen, since we don't abort any request until
* we received all related PDUs from the initiator or timeout
* them. Let's quietly drop such PDUs.
*/
TRACE_MGMT_DBG("Unable to find scsi task ITT %x",
cmnd->pdu.bhs.itt);
res = iscsi_preliminary_complete(cmnd, cmnd, true);
goto out;
}
cmnd_get(orig_req);
if (unlikely(orig_req->r2t_len_to_receive < cmnd->pdu.datasize)) {
if (orig_req->prelim_compl_flags != 0) {
/* We can have fake r2t_len_to_receive */
goto go;
}
PRINT_ERROR("Data size (%d) > R2T length to receive (%d)",
cmnd->pdu.datasize, orig_req->r2t_len_to_receive);
set_scst_preliminary_status_rsp(orig_req, false,
SCST_LOAD_SENSE(iscsi_sense_incorrect_amount_of_data));
goto go;
}
/* Crazy iSCSI spec requires us to make this unneeded check */
#if 0 /* ...but some initiators (Windows) don't care to correctly set it */
orig_req_hdr = &orig_req->pdu.bhs;
if (unlikely(orig_req_hdr->lun != req_hdr->lun)) {
PRINT_ERROR("Wrong LUN (%lld) in Data-Out PDU (expected %lld), orig_req %p, cmnd %p",
(unsigned long long)req_hdr->lun,
(unsigned long long)orig_req_hdr->lun, orig_req, cmnd);
create_reject_rsp(orig_req, ISCSI_REASON_PROTOCOL_ERROR, false);
goto go;
}
#endif
go:
if (req_hdr->flags & ISCSI_FLG_FINAL)
orig_req->outstanding_r2t--;
EXTRACHECKS_BUG_ON(orig_req->data_out_in_data_receiving);
orig_req->data_out_in_data_receiving = 1;
TRACE_WRITE("cmnd %p, orig_req %p, offset %u, datasize %u", cmnd,
orig_req, offset, cmnd->pdu.datasize);
if (unlikely(orig_req->prelim_compl_flags != 0))
res = iscsi_preliminary_complete(cmnd, orig_req, true);
else
res = cmnd_prepare_recv_pdu(conn, orig_req, offset,
cmnd->pdu.datasize);
out:
TRACE_EXIT_RES(res);
return res;
}
static void data_out_end(struct iscsi_cmnd *cmnd)
{
struct iscsi_data_out_hdr *req_hdr =
(struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
struct iscsi_cmnd *req;
TRACE_ENTRY();
EXTRACHECKS_BUG_ON(!cmnd);
req = cmnd->cmd_req;
if (unlikely(!req))
goto out;
TRACE_DBG("cmnd %p, req %p", cmnd, req);
iscsi_extracheck_is_rd_thread(cmnd->conn);
req->data_out_in_data_receiving = 0;
if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
!cmnd->ddigest_checked) {
cmd_add_on_rx_ddigest_list(req, cmnd);
cmnd_get(cmnd);
}
/*
* Now we received the data and can adjust r2t_len_to_receive of the
* orig req. We couldn't do it earlier, because it will break data
* receiving errors recovery (calls of iscsi_fail_data_waiting_cmnd()).
*/
req->r2t_len_to_receive -= cmnd->pdu.datasize;
if (unlikely(req->prelim_compl_flags != 0)) {
/*
* We need to call iscsi_preliminary_complete() again
* to handle the case if we just been aborted. This call must
* be done before zeroing r2t_len_to_send to correctly calc.
* residual.
*/
iscsi_preliminary_complete(cmnd, req, false);
/*
* We might need to wait for one or more PDUs. Let's simplify
* other code and not perform exact r2t_len_to_receive
* calculation.
*/
req->r2t_len_to_receive = req->outstanding_r2t;
req->r2t_len_to_send = 0;
}
TRACE_DBG("req %p, FINAL %x, outstanding_r2t %d, r2t_len_to_receive %d, r2t_len_to_send %d",
req, req_hdr->flags & ISCSI_FLG_FINAL,
req->outstanding_r2t, req->r2t_len_to_receive,
req->r2t_len_to_send);
if (!(req_hdr->flags & ISCSI_FLG_FINAL))
goto out_put;
if (req->r2t_len_to_receive == 0) {
if (!req->pending)
iscsi_restart_cmnd(req);
} else if (req->r2t_len_to_send != 0) {
send_r2t(req);
}
out_put:
cmnd_put(req);
cmnd->cmd_req = NULL;
out:
TRACE_EXIT();
}
/* Might be called under target_mutex and cmd_list_lock */
static void __cmnd_abort(struct iscsi_cmnd *cmnd)
{
unsigned long timeout_time = jiffies + ISCSI_TM_DATA_WAIT_TIMEOUT +
ISCSI_ADD_SCHED_TIME;
struct iscsi_conn *conn = cmnd->conn;
#ifdef CONFIG_SCST_EXTRACHECKS
struct task_struct *rdt = cmnd->conn->rd_task;
#else
struct task_struct *rdt = NULL;
#endif
TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, ref_cnt %d, on_write_timeout_list %d, write_start %ld, ITT %x, sn %u, op %x, r2t_len_to_receive %d, r2t_len_to_send %d, CDB op %x, size to write %u, outstanding_r2t %d, sess->exp_cmd_sn %u, conn %p, rd_task %p, read_cmnd %p, read_state %d)",
cmnd, cmnd->scst_cmd, cmnd->scst_state,
atomic_read(&cmnd->ref_cnt), cmnd->on_write_timeout_list,
cmnd->write_start, cmnd->pdu.bhs.itt, cmnd->pdu.bhs.sn,
cmnd_opcode(cmnd), cmnd->r2t_len_to_receive,
cmnd->r2t_len_to_send, cmnd_scsicode(cmnd),
cmnd_write_size(cmnd), cmnd->outstanding_r2t,
cmnd->conn->session->exp_cmd_sn, cmnd->conn,
rdt, cmnd->conn->read_cmnd, cmnd->conn->read_state);
/*
* Lock to sync with iscsi_check_tm_data_wait_timeouts(), including
* CMD_ABORTED bit set.
*/
spin_lock_bh(&conn->conn_thr_pool->rd_lock);
/*
* We suppose that preliminary commands completion is tested by
* comparing prelim_compl_flags with 0. Otherwise a race is possible,
* like sending command in SCST core as PRELIM_COMPLETED, while it
* wasn't aborted in it yet and have as the result a wrong success
* status sent to the initiator.
*/
set_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags);
TRACE_MGMT_DBG("Setting conn_tm_active for conn %p", conn);
conn->conn_tm_active = 1;
spin_unlock_bh(&conn->conn_thr_pool->rd_lock);
/*
* We need the lock to sync with req_add_to_write_timeout_list() and
* close races for rsp_timer.expires.
*/
spin_lock_bh(&conn->write_list_lock);
if (!timer_pending(&conn->rsp_timer) ||
time_after(conn->rsp_timer.expires, timeout_time)) {
TRACE_MGMT_DBG("Mod timer on %ld (conn %p)",
timeout_time, conn);
mod_timer(&conn->rsp_timer, timeout_time);
} else {
TRACE_MGMT_DBG("Timer for conn %p is going to fire on %ld (timeout time %ld)",
conn, conn->rsp_timer.expires, timeout_time);
}
spin_unlock_bh(&conn->write_list_lock);
}
/* Must be called from the read or conn close thread */
static int cmnd_abort_pre_checks(struct iscsi_cmnd *req, int *status)
{
struct iscsi_task_mgt_hdr *req_hdr =
(struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
struct iscsi_cmnd *cmnd;
int res = -1;
req_hdr->ref_cmd_sn = be32_to_cpu((__force __be32)req_hdr->ref_cmd_sn);
if (!before(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
TRACE(TRACE_MGMT, "ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
*status = ISCSI_RESPONSE_UNKNOWN_TASK;
goto out;
}
cmnd = cmnd_find_itt_get(req->conn, req_hdr->rtt);
if (cmnd) {
struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
if (req_hdr->lun != hdr->lun) {
PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN %llx, cmd LUN %llx, rtt %u",
be64_to_cpu(req_hdr->lun),
be64_to_cpu(hdr->lun),
req_hdr->rtt);
*status = ISCSI_RESPONSE_FUNCTION_REJECTED;
goto out_put;
}
if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM cmd CmdSN(%u) for immediate command %p",
req_hdr->ref_cmd_sn, req_hdr->cmd_sn, cmnd);
*status = ISCSI_RESPONSE_FUNCTION_REJECTED;
goto out_put;
}
} else {
if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != CmdSN(%u) for command %p",
req_hdr->ref_cmd_sn, req_hdr->cmd_sn, cmnd);
*status = ISCSI_RESPONSE_FUNCTION_REJECTED;
goto out_put;
}
}
if (before(req_hdr->cmd_sn, hdr->cmd_sn) || req_hdr->cmd_sn == hdr->cmd_sn) {
PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, cmd SN %x, rtt %u",
req_hdr->cmd_sn, hdr->cmd_sn, req_hdr->rtt);
*status = ISCSI_RESPONSE_FUNCTION_REJECTED;
goto out_put;
}
cmnd_put(cmnd);
res = 0;
} else {
TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
/*
* iSCSI RFC:
*
* b) If the Referenced Task Tag does not identify an existing
* task, but if the CmdSN indicated by the RefCmdSN field in
* the Task Management function request is within the valid
* CmdSN window and less than the CmdSN of the Task Management
* function request itself, then targets must consider the
* CmdSN received and return the "Function complete" response.
*
* c) If the Referenced Task Tag does not identify an existing
* task and if the CmdSN indicated by the RefCmdSN field in
* the Task Management function request is outside the valid
* CmdSN window, then targets must return the "Task does not
* exist" response.
*
* 2048 seems to be a good "window".
*/
if (between(req_hdr->ref_cmd_sn, req_hdr->cmd_sn - 2048,
req_hdr->cmd_sn)) {
*status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
res = 0;
} else {
*status = ISCSI_RESPONSE_UNKNOWN_TASK;
}
}
out:
return res;
out_put:
cmnd_put(cmnd);
goto out;
}
struct iscsi_cmnd_abort_params {
struct work_struct iscsi_cmnd_abort_work;
struct scst_cmd *scst_cmd;
};
static mempool_t *iscsi_cmnd_abort_mempool;
static void iscsi_cmnd_abort_fn(struct work_struct *work)
{
struct iscsi_cmnd_abort_params *params = container_of(work,
struct iscsi_cmnd_abort_params, iscsi_cmnd_abort_work);
struct scst_cmd *scst_cmd = params->scst_cmd;
struct iscsi_session *session = scst_sess_get_tgt_priv(scst_cmd->sess);
struct iscsi_conn *conn;
struct iscsi_cmnd *cmnd = scst_cmd_get_tgt_priv(scst_cmd);
bool done = false;
TRACE_ENTRY();
TRACE_MGMT_DBG("Checking aborted scst_cmd %p (cmnd %p)", scst_cmd,
cmnd);
mutex_lock(&session->target->target_mutex);
/*
* cmnd pointer is valid only under cmd_list_lock, but we can't know the
* corresponding conn without dereferencing cmnd at first, so let's
* check all conns and cmnds to find out if our cmnd is still valid
* under lock.
*/
list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
struct iscsi_cmnd *c;
spin_lock_bh(&conn->cmd_list_lock);
list_for_each_entry(c, &conn->cmd_list, cmd_list_entry) {
if (c == cmnd) {
__cmnd_abort(cmnd);
done = true;
break;
}
}
spin_unlock_bh(&conn->cmd_list_lock);
if (done)
break;
}
mutex_unlock(&session->target->target_mutex);
scst_cmd_put(scst_cmd);
mempool_free(params, iscsi_cmnd_abort_mempool);
TRACE_EXIT();
}
static void iscsi_on_abort_cmd(struct scst_cmd *scst_cmd)
{
struct iscsi_cmnd_abort_params *params;
TRACE_ENTRY();
params = mempool_alloc(iscsi_cmnd_abort_mempool, GFP_ATOMIC);
if (!params) {
PRINT_CRIT_ERROR("Unable to create iscsi_cmnd_abort_params, iSCSI cmnd for scst_cmd %p may not be aborted",
scst_cmd);
goto out;
}
memset(params, 0, sizeof(*params));
INIT_WORK(&params->iscsi_cmnd_abort_work, iscsi_cmnd_abort_fn);
params->scst_cmd = scst_cmd;
scst_cmd_get(scst_cmd);
TRACE_MGMT_DBG("Scheduling abort check for scst_cmd %p", scst_cmd);
schedule_work(&params->iscsi_cmnd_abort_work);
out:
TRACE_EXIT();
}
/* Must be called from the read or conn close thread */
void conn_abort(struct iscsi_conn *conn)
{
struct iscsi_cmnd *cmnd, *r, *t;
TRACE_MGMT_DBG("Aborting conn %p", conn);
iscsi_extracheck_is_rd_thread(conn);
cancel_delayed_work_sync(&conn->nop_in_delayed_work);
/* No locks, we are the only user */
list_for_each_entry_safe(r, t, &conn->nop_req_list, nop_req_list_entry) {
list_del(&r->nop_req_list_entry);
cmnd_put(r);
}
spin_lock_bh(&conn->cmd_list_lock);
again:
list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
__cmnd_abort(cmnd);
if (cmnd->r2t_len_to_receive != 0) {
if (!cmnd_get_check(cmnd)) {
spin_unlock_bh(&conn->cmd_list_lock);
/* ToDo: this is racy for MC/S */
iscsi_fail_data_waiting_cmnd(cmnd);
cmnd_put(cmnd);
/*
* We are in the read thread, so we may not
* worry that after cmnd release conn gets
* released as well.
*/
spin_lock_bh(&conn->cmd_list_lock);
goto again;
}
}
}
spin_unlock_bh(&conn->cmd_list_lock);
}
/*
* Drops, but not releases(!) delayed TM response.
* sn_lock supposed to be held on entry.
*/
void iscsi_drop_delayed_tm_rsp(struct iscsi_cmnd *tm_rsp)
{
struct iscsi_conn *conn = tm_rsp->conn;
struct iscsi_session *sess = conn->session;
TRACE_ENTRY();
TRACE_MGMT_DBG("Dropping delayed TM rsp %p", tm_rsp);
sess->tm_rsp = NULL;
sess->tm_active--;
sBUG_ON(sess->tm_active < 0);
TRACE_EXIT();
}
static void execute_task_management(struct iscsi_cmnd *req)
{
struct iscsi_conn *conn = req->conn;
struct iscsi_session *sess = conn->session;
struct iscsi_task_mgt_hdr *req_hdr =
(struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
int rc, status = ISCSI_RESPONSE_FUNCTION_REJECTED;
int function = req_hdr->function & ISCSI_FUNCTION_MASK;
struct scst_rx_mgmt_params params;
TRACE(TRACE_MGMT, "iSCSI TM fn %d", function);
TRACE_MGMT_DBG("TM req %p, ITT %x, RTT %x, sn %u, con %p",
req, req->pdu.bhs.itt, req_hdr->rtt, req_hdr->cmd_sn, conn);
iscsi_extracheck_is_rd_thread(conn);
spin_lock(&sess->sn_lock);
sess->tm_active++;
sess->tm_sn = req_hdr->cmd_sn;
spin_unlock(&sess->sn_lock);
scst_rx_mgmt_params_init(&params);
params.atomic = SCST_NON_ATOMIC;
params.tgt_priv = req;
if (function != ISCSI_FUNCTION_ABORT_TASK && req_hdr->rtt != ISCSI_RESERVED_TAG) {
PRINT_ERROR("Invalid RTT %x (TM fn %d)",
req_hdr->rtt, function);
rc = -1;
status = ISCSI_RESPONSE_FUNCTION_REJECTED;
goto reject;
}
/* cmd_sn is already in CPU format converted in cmnd_rx_start() */
switch (function) {
case ISCSI_FUNCTION_ABORT_TASK:
rc = cmnd_abort_pre_checks(req, &status);
if (rc == 0) {
params.fn = SCST_ABORT_TASK;
params.tag = (__force u32)req_hdr->rtt;
params.tag_set = 1;
params.lun = (uint8_t *)&req_hdr->lun;
params.lun_len = sizeof(req_hdr->lun);
params.lun_set = 1;
params.cmd_sn = req_hdr->cmd_sn;
params.cmd_sn_set = 1;
rc = scst_rx_mgmt_fn(conn->session->scst_sess, &params);
status = ISCSI_RESPONSE_FUNCTION_REJECTED;
}
break;
case ISCSI_FUNCTION_ABORT_TASK_SET:
params.fn = SCST_ABORT_TASK_SET;
params.lun = (uint8_t *)&req_hdr->lun;
params.lun_len = sizeof(req_hdr->lun);
params.lun_set = 1;
params.cmd_sn = req_hdr->cmd_sn;
params.cmd_sn_set = 1;
rc = scst_rx_mgmt_fn(conn->session->scst_sess, &params);
status = ISCSI_RESPONSE_FUNCTION_REJECTED;
break;
case ISCSI_FUNCTION_CLEAR_TASK_SET:
params.fn = SCST_CLEAR_TASK_SET;
params.lun = (uint8_t *)&req_hdr->lun;
params.lun_len = sizeof(req_hdr->lun);
params.lun_set = 1;
params.cmd_sn = req_hdr->cmd_sn;
params.cmd_sn_set = 1;
rc = scst_rx_mgmt_fn(conn->session->scst_sess, &params);
status = ISCSI_RESPONSE_FUNCTION_REJECTED;
break;
case ISCSI_FUNCTION_CLEAR_ACA:
params.fn = SCST_CLEAR_ACA;
params.lun = (uint8_t *)&req_hdr->lun;
params.lun_len = sizeof(req_hdr->lun);
params.lun_set = 1;
params.cmd_sn = req_hdr->cmd_sn;
params.cmd_sn_set = 1;
rc = scst_rx_mgmt_fn(conn->session->scst_sess, &params);
status = ISCSI_RESPONSE_FUNCTION_REJECTED;
break;
case ISCSI_FUNCTION_TARGET_COLD_RESET:
case ISCSI_FUNCTION_TARGET_WARM_RESET:
params.fn = SCST_TARGET_RESET;
params.cmd_sn = req_hdr->cmd_sn;
params.cmd_sn_set = 1;
rc = scst_rx_mgmt_fn(conn->session->scst_sess, &params);
status = ISCSI_RESPONSE_FUNCTION_REJECTED;
break;
case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
params.fn = SCST_LUN_RESET;
params.lun = (uint8_t *)&req_hdr->lun;
params.lun_len = sizeof(req_hdr->lun);
params.lun_set = 1;
params.cmd_sn = req_hdr->cmd_sn;
params.cmd_sn_set = 1;
rc = scst_rx_mgmt_fn(conn->session->scst_sess, &params);
status = ISCSI_RESPONSE_FUNCTION_REJECTED;
break;
case ISCSI_FUNCTION_TASK_REASSIGN:
rc = -1;
status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
break;
default:
PRINT_ERROR("Unknown TM function %d", function);
rc = -1;
status = ISCSI_RESPONSE_FUNCTION_REJECTED;
break;
}
reject:
if (rc != 0)
iscsi_send_task_mgmt_resp(req, status, false);
}
static void iscsi_tcp_set_req_data(struct iscsi_cmnd *req,
struct iscsi_cmnd *rsp)
{
rsp->sg = req->sg;
rsp->sg_cnt = req->sg_cnt;
rsp->bufflen = req->bufflen;
}
static void nop_out_exec(struct iscsi_cmnd *req)
{
struct iscsi_cmnd *rsp;
struct iscsi_nop_in_hdr *rsp_hdr;
TRACE_ENTRY();
TRACE_DBG("%p", req);
if (req->pdu.bhs.itt != ISCSI_RESERVED_TAG) {
rsp = iscsi_alloc_main_rsp(req);
rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_NOP_IN;
rsp_hdr->flags = ISCSI_FLG_FINAL;
rsp_hdr->itt = req->pdu.bhs.itt;
rsp_hdr->ttt = ISCSI_RESERVED_TAG;
if (req->pdu.datasize)
sBUG_ON(!req->sg);
else
sBUG_ON(req->sg);
if (req->bufflen)
req->conn->transport->iscsit_set_req_data(req, rsp);
/* We already checked it in check_segment_length() */
sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
rsp->pdu.datasize = req->pdu.datasize;
} else {
bool found = false;
struct iscsi_cmnd *r;
struct iscsi_conn *conn = req->conn;
TRACE_DBG("Receive Nop-In response (ttt 0x%08x)",
be32_to_cpu(req->pdu.bhs.ttt));
spin_lock_bh(&conn->nop_req_list_lock);
list_for_each_entry(r, &conn->nop_req_list, nop_req_list_entry) {
if (req->pdu.bhs.ttt == r->pdu.bhs.ttt) {
list_del(&r->nop_req_list_entry);
found = true;
break;
}
}
spin_unlock_bh(&conn->nop_req_list_lock);
if (found)
cmnd_put(r);
else
TRACE_MGMT_DBG("Got Nop-out response without corresponding Nop-In request");
}
req_cmnd_release(req);
TRACE_EXIT();
}
static void logout_exec(struct iscsi_cmnd *req)
{
struct iscsi_logout_req_hdr *req_hdr;
struct iscsi_cmnd *rsp;
struct iscsi_logout_rsp_hdr *rsp_hdr;
PRINT_INFO("Logout received from initiator %s",
req->conn->session->initiator_name);
TRACE_DBG("%p", req);
req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
rsp = iscsi_alloc_main_rsp(req);
rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
rsp_hdr->flags = ISCSI_FLG_FINAL;
rsp_hdr->itt = req_hdr->itt;
rsp->should_close_conn = 1;
req_cmnd_release(req);
}
static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
{
TRACE_ENTRY();
TRACE_DBG("cmnd %p, op %x, SN %u",
cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
iscsi_extracheck_is_rd_thread(cmnd->conn);
if (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD) {
if (cmnd->r2t_len_to_receive == 0)
iscsi_restart_cmnd(cmnd);
else if (cmnd->r2t_len_to_send != 0)
send_r2t(cmnd);
goto out;
}
if (cmnd->prelim_compl_flags != 0) {
TRACE_MGMT_DBG("Terminating prelim completed non-SCSI cmnd %p (op %x)",
cmnd, cmnd_opcode(cmnd));
req_cmnd_release(cmnd);
goto out;
}
switch (cmnd_opcode(cmnd)) {
case ISCSI_OP_NOP_OUT:
nop_out_exec(cmnd);
break;
case ISCSI_OP_SCSI_TASK_MGT_MSG:
execute_task_management(cmnd);
break;
case ISCSI_OP_LOGOUT_CMD:
logout_exec(cmnd);
break;
default:
PRINT_CRIT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
sBUG();
break;
}
out:
TRACE_EXIT();
}
static inline void set_cork(struct socket *sock, int on)
{
int opt = on;
mm_segment_t oldfs;
oldfs = get_fs();
set_fs(KERNEL_DS);
sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
KERNEL_SOCKPTR(&opt), sizeof(opt));
set_fs(oldfs);
}
void cmnd_tx_start(struct iscsi_cmnd *cmnd)
{
struct iscsi_conn *conn = cmnd->conn;
TRACE_DBG("conn %p, cmnd %p, opcode %x", conn, cmnd, cmnd_opcode(cmnd));
iscsi_cmnd_set_length(&cmnd->pdu);
iscsi_extracheck_is_wr_thread(conn);
set_cork(conn->sock, 1);
conn->write_iop = conn->write_iov;
conn->write_iop->iov_base = &cmnd->pdu.bhs;
conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
conn->write_iop_used = 1;
conn->write_size = sizeof(cmnd->pdu.bhs) + cmnd->pdu.datasize;
conn->write_offset = 0;
switch (cmnd_opcode(cmnd)) {
case ISCSI_OP_NOP_IN:
if (cmnd->pdu.bhs.itt == ISCSI_RESERVED_TAG)
cmnd->pdu.bhs.sn = (__force u32)cmnd_set_sn(cmnd, 0);
else
cmnd_set_sn(cmnd, 1);
break;
case ISCSI_OP_SCSI_RSP:
cmnd_set_sn(cmnd, 1);
break;
case ISCSI_OP_SCSI_TASK_MGT_RSP:
cmnd_set_sn(cmnd, 1);
break;
case ISCSI_OP_TEXT_RSP:
cmnd_set_sn(cmnd, 1);
break;
case ISCSI_OP_SCSI_DATA_IN:
{
struct iscsi_data_in_hdr *rsp =
(struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
u32 offset = be32_to_cpu(rsp->buffer_offset);
TRACE_DBG("cmnd %p, offset %u, datasize %u, bufflen %u",
cmnd, offset, cmnd->pdu.datasize, cmnd->bufflen);
sBUG_ON(offset > cmnd->bufflen);
sBUG_ON(offset + cmnd->pdu.datasize > cmnd->bufflen);
conn->write_offset = offset;
cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
break;
}
case ISCSI_OP_LOGOUT_RSP:
cmnd_set_sn(cmnd, 1);
break;
case ISCSI_OP_R2T:
cmnd->pdu.bhs.sn = (__force u32)cmnd_set_sn(cmnd, 0);
break;
case ISCSI_OP_ASYNC_MSG:
cmnd_set_sn(cmnd, 1);
break;
case ISCSI_OP_REJECT:
cmnd_set_sn(cmnd, 1);
break;
default:
PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
break;
}
iscsi_dump_pdu(&cmnd->pdu);
}
void cmnd_tx_end(struct iscsi_cmnd *cmnd)
{
struct iscsi_conn *conn = cmnd->conn;
TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
cmnd->should_close_all_conn);
#ifdef CONFIG_SCST_EXTRACHECKS
switch (cmnd_opcode(cmnd)) {
case ISCSI_OP_NOP_IN:
case ISCSI_OP_SCSI_RSP:
case ISCSI_OP_SCSI_TASK_MGT_RSP:
case ISCSI_OP_TEXT_RSP:
case ISCSI_OP_R2T:
case ISCSI_OP_ASYNC_MSG:
case ISCSI_OP_REJECT:
case ISCSI_OP_SCSI_DATA_IN:
case ISCSI_OP_LOGOUT_RSP:
break;
default:
PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
sBUG();
break;
}
#endif
if (unlikely(cmnd->should_close_conn)) {
if (cmnd->should_close_all_conn) {
struct iscsi_target *target = cmnd->conn->session->target;
PRINT_INFO("Closing all connections for target %x at initiator's %s request",
target->tid, conn->session->initiator_name);
mutex_lock(&target->target_mutex);
target_del_all_sess(target, 0);
mutex_unlock(&target->target_mutex);
} else {
PRINT_INFO("Closing connection at initiator's %s request",
conn->session->initiator_name);
mark_conn_closed(conn);
}
}
set_cork(cmnd->conn->sock, 0);
}
/*
* Push the command for execution. This functions reorders the commands.
* Called from the read thread.
*
* Basically, since we don't support MC/S and TCP guarantees data delivery
* order, all that SN's stuff isn't needed at all (commands delivery order is
* a natural commands execution order), but insane iSCSI spec requires
* us to check it and we have to, because some crazy initiators can rely
* on the SN's based order and reorder requests during sending. For all other
* normal initiators all that code is a NOP.
*/
static void iscsi_push_cmnd(struct iscsi_cmnd *cmnd)
{
struct iscsi_session *session = cmnd->conn->session;
struct list_head *entry;
u32 cmd_sn;
TRACE_DBG("cmnd %p, iSCSI opcode %x, sn %u, exp sn %u", cmnd,
cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
iscsi_extracheck_is_rd_thread(cmnd->conn);
sBUG_ON(cmnd->parent_req);
if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
cmnd->pdu.bhs.sn);
iscsi_cmnd_exec(cmnd);
return;
}
spin_lock(&session->sn_lock);
cmd_sn = cmnd->pdu.bhs.sn;
if (cmd_sn == session->exp_cmd_sn) {
while (1) {
session->exp_cmd_sn = ++cmd_sn;
if (unlikely(session->tm_active > 0)) {
if (before(cmd_sn, session->tm_sn)) {
struct iscsi_conn *conn = cmnd->conn;
spin_unlock(&session->sn_lock);
spin_lock_bh(&conn->cmd_list_lock);
__cmnd_abort(cmnd);
spin_unlock_bh(&conn->cmd_list_lock);
spin_lock(&session->sn_lock);
}
iscsi_check_send_delayed_tm_resp(session);
}
spin_unlock(&session->sn_lock);
iscsi_cmnd_exec(cmnd);
spin_lock(&session->sn_lock);
if (list_empty(&session->pending_list))
break;
cmnd = list_first_entry(&session->pending_list, struct iscsi_cmnd,
pending_list_entry);
if (cmnd->pdu.bhs.sn != cmd_sn)
break;
list_del(&cmnd->pending_list_entry);
cmnd->pending = 0;
TRACE_MGMT_DBG("Processing pending cmd %p (cmd_sn %u)",
cmnd, cmd_sn);
}
} else {
int drop = 0;
TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
cmnd, cmd_sn, session->exp_cmd_sn);
/*
* iSCSI RFC 3720: "The target MUST silently ignore any
* non-immediate command outside of [from ExpCmdSN to MaxCmdSN
* inclusive] range". But we won't honor the MaxCmdSN
* requirement, because, since we adjust MaxCmdSN from the
* separate write thread, rarely it is possible that initiator
* can legally send command with CmdSN>MaxSN. But it won't
* hurt anything, in the worst case it will lead to
* additional QUEUE FULL status.
*/
if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
TRACE_MGMT_DBG("Ignoring out of expected range cmd_sn (sn %u, exp_sn %u, cmd %p, op %x, CDB op %x)",
cmd_sn, session->exp_cmd_sn, cmnd,
cmnd_opcode(cmnd), cmnd_scsicode(cmnd));
drop = 1;
}
#if 0
if (unlikely(after(cmd_sn, session->exp_cmd_sn +
iscsi_get_allowed_cmds(session)))) {
TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, max_sn %u)",
cmd_sn, session->exp_cmd_sn,
iscsi_get_allowed_cmds(session));
drop = 1;
}
#endif
spin_unlock(&session->sn_lock);
if (unlikely(drop)) {
req_cmnd_release_force(cmnd);
return;
}
if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
struct iscsi_cmnd *tm_clone;
TRACE_MGMT_DBG("Aborted pending cmnd %p, creating TM clone (scst cmd %p, state %d)",
cmnd, cmnd->scst_cmd, cmnd->scst_state);
tm_clone = iscsi_create_tm_clone(cmnd);
if (tm_clone) {
iscsi_cmnd_exec(cmnd);
cmnd = tm_clone;
}
}
TRACE_MGMT_DBG("Pending cmnd %p (op %x, sn %u, exp sn %u)",
cmnd, cmnd_opcode(cmnd), cmd_sn, session->exp_cmd_sn);
spin_lock(&session->sn_lock);
list_for_each(entry, &session->pending_list) {
struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
pending_list_entry);
if (before(cmd_sn, tmp->pdu.bhs.sn))
break;
}
list_add_tail(&cmnd->pending_list_entry, entry);
cmnd->pending = 1;
}
spin_unlock(&session->sn_lock);
}
static int check_segment_length(struct iscsi_cmnd *cmnd)
{
struct iscsi_conn *conn = cmnd->conn;
struct iscsi_session *session = conn->session;
if (unlikely(cmnd->pdu.datasize > session->sess_params.max_recv_data_length)) {
PRINT_ERROR("Initiator %s (conn %p) violated negotiated parameters: data too long (ITT %x, datasize %u, max_recv_data_length %u)",
session->initiator_name,
conn, cmnd->pdu.bhs.itt, cmnd->pdu.datasize,
session->sess_params.max_recv_data_length);
mark_conn_closed(conn);
return -EINVAL;
}
return 0;
}
int cmnd_rx_start(struct iscsi_cmnd *cmnd)
{
int res, rc = 0;
iscsi_dump_pdu(&cmnd->pdu);
res = check_segment_length(cmnd);
if (res != 0)
goto out;
cmnd->pdu.bhs.sn = be32_to_cpu((__force __be32)cmnd->pdu.bhs.sn);
switch (cmnd_opcode(cmnd)) {
case ISCSI_OP_SCSI_CMD:
res = scsi_cmnd_start(cmnd);
if (unlikely(res < 0))
goto out;
update_stat_sn(cmnd);
break;
case ISCSI_OP_SCSI_DATA_OUT:
res = data_out_start(cmnd);
goto out;
case ISCSI_OP_NOP_OUT:
rc = nop_out_start(cmnd);
break;
case ISCSI_OP_SCSI_TASK_MGT_MSG:
case ISCSI_OP_LOGOUT_CMD:
update_stat_sn(cmnd);
break;
case ISCSI_OP_TEXT_CMD:
case ISCSI_OP_SNACK_CMD:
default:
rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
break;
}
if (unlikely(rc < 0)) {
PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x)",
rc, cmnd_opcode(cmnd), cmnd->pdu.bhs.itt);
res = create_reject_rsp(cmnd, -rc, true);
}
out:
TRACE_EXIT_RES(res);
return res;
}
EXPORT_SYMBOL(cmnd_rx_start);
void cmnd_rx_end(struct iscsi_cmnd *cmnd)
{
TRACE_ENTRY();
TRACE_DBG("cmnd %p, opcode %x", cmnd, cmnd_opcode(cmnd));
cmnd->conn->last_rcv_time = jiffies;
TRACE_DBG("Updated last_rcv_time %ld", cmnd->conn->last_rcv_time);
switch (cmnd_opcode(cmnd)) {
case ISCSI_OP_SCSI_CMD:
case ISCSI_OP_NOP_OUT:
case ISCSI_OP_SCSI_TASK_MGT_MSG:
case ISCSI_OP_LOGOUT_CMD:
iscsi_push_cmnd(cmnd);
goto out;
case ISCSI_OP_SCSI_DATA_OUT:
data_out_end(cmnd);
break;
default:
PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
break;
}
req_cmnd_release(cmnd);
out:
TRACE_EXIT();
}
EXPORT_SYMBOL(cmnd_rx_end);
static ssize_t iscsi_tcp_get_initiator_ip(struct iscsi_conn *conn, char *buf, int size)
{
int pos;
struct sock *sk;
TRACE_ENTRY();
sk = conn->sock->sk;
switch (sk->sk_family) {
case AF_INET:
pos = scnprintf(buf, size, "%pI4", &inet_sk(sk)->inet_daddr);
break;
#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
case AF_INET6:
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0) && \
(!defined(RHEL_MAJOR) || RHEL_MAJOR -0 < 7)
pos = scnprintf(buf, size, "[%pI6]", &inet6_sk(sk)->daddr);
#else
pos = scnprintf(buf, size, "[%pI6]", &sk->sk_v6_daddr);
#endif
#endif
break;
default:
pos = scnprintf(buf, size, "Unknown family %d", sk->sk_family);
break;
}
TRACE_EXIT_RES(pos);
return pos;
}
static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
{
/*
* sock->ops->sendpage() is async zero copy operation,
* so we must be sure not to free and reuse
* the command's buffer before the sending was completed
* by the network layers. It is possible only if we
* don't use SGV cache.
*/
EXTRACHECKS_BUG_ON(!(scst_cmd_get_data_direction(cmd) &
SCST_DATA_READ));
scst_cmd_set_no_sgv(cmd);
return 1;
}
static void iscsi_tcp_preprocessing_done(struct iscsi_cmnd *req)
{
TRACE_DBG("req %p", req);
if (req->conn->rx_task == current) {
req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
} else {
/*
* We wait for the state change without any protection, so
* without cmnd_get() it is possible that req will die
* "immediately" after the state assignment and
* iscsi_make_conn_rd_active() will operate on dead data.
* We use the ordered version of cmnd_get(), because "get"
* must be done before the state assignment.
*
* We protected from the race on calling cmnd_rx_continue(),
* because there can be only one read thread processing
* connection.
*/
cmnd_get(req);
req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
iscsi_make_conn_rd_active(req->conn);
if (unlikely(req->conn->closing)) {
TRACE_DBG("Waking up closing conn %p", req->conn);
wake_up(&req->conn->read_state_waitQ);
}
cmnd_put(req);
}
}
static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
{
struct iscsi_cmnd *req = scst_cmd_get_tgt_priv(scst_cmd);
req->conn->transport->iscsit_preprocessing_done(req);
}
/* No locks */
static void iscsi_try_local_processing(struct iscsi_cmnd *req)
{
struct iscsi_conn *conn = req->conn;
struct iscsi_thread_pool *p = conn->conn_thr_pool;
bool local;
TRACE_ENTRY();
spin_lock_bh(&p->wr_lock);
switch (conn->wr_state) {
case ISCSI_CONN_WR_STATE_IN_LIST:
list_del(&conn->wr_list_entry);
fallthrough;
case ISCSI_CONN_WR_STATE_IDLE:
#ifdef CONFIG_SCST_EXTRACHECKS
conn->wr_task = current;
#endif
conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
conn->wr_space_ready = 0;
local = true;
break;
default:
local = false;
break;
}
spin_unlock_bh(&p->wr_lock);
if (local) {
int rc = 1;
do {
rc = iscsi_send(conn);
if (rc <= 0)
break;
} while (req->not_processed_rsp_cnt != 0);
spin_lock_bh(&p->wr_lock);
#ifdef CONFIG_SCST_EXTRACHECKS
conn->wr_task = NULL;
#endif
if ((rc == -EAGAIN) && !conn->wr_space_ready) {
TRACE_DBG("EAGAIN, setting WR_STATE_SPACE_WAIT (conn %p)",
conn);
conn->wr_state = ISCSI_CONN_WR_STATE_SPACE_WAIT;
} else if (test_write_ready(conn)) {
list_add_tail(&conn->wr_list_entry, &p->wr_list);
conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
wake_up(&p->wr_waitQ);
} else {
conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
}
spin_unlock_bh(&p->wr_lock);
}
TRACE_EXIT();
}
static int iscsi_tcp_send_locally(struct iscsi_cmnd *req,
unsigned int cmd_count)
{
struct iscsi_conn *conn = req->conn;
struct iscsi_cmnd *wr_rsp, *our_rsp;
int ret = 0;
/*
* There's no need for protection, since we are not going to
* dereference them.
*/
wr_rsp = list_first_entry(&conn->write_list, struct iscsi_cmnd, write_list_entry);
our_rsp = list_first_entry(&req->rsp_cmd_list, struct iscsi_cmnd, rsp_cmd_list_entry);
if (wr_rsp == our_rsp) {
/*
* This is our rsp, so let's try to process it locally to
* decrease latency. We need to call pre_release before
* processing to handle some error recovery cases.
*/
if (cmd_count <= 2) {
req_cmnd_pre_release(req);
iscsi_try_local_processing(req);
cmnd_put(req);
} else {
/*
* There's too much backend activity, so it could be
* better to push it to the write thread.
*/
ret = 1;
}
} else {
ret = 1;
}
return ret;
}
static void iscsi_tcp_conn_close(struct iscsi_conn *conn, int flags)
{
struct sock *sk = conn->sock->sk;
if (!flags) {
lock_sock(sk);
sk->sk_prot->disconnect(sk, 0);
release_sock(sk);
} else {
conn->sock->ops->shutdown(conn->sock, flags);
}
}
static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
{
int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
struct iscsi_cmnd *req = scst_cmd_get_tgt_priv(scst_cmd);
struct iscsi_conn *conn = req->conn;
int status = scst_cmd_get_status(scst_cmd);
u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
scst_cmd_set_tgt_priv(scst_cmd, NULL);
EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RESTARTED);
if (unlikely(scst_cmd_aborted_on_xmit(scst_cmd)))
set_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags);
if (unlikely(req->prelim_compl_flags != 0)) {
if (test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags)) {
TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted",
req, req->scst_cmd);
scst_set_delivery_status(req->scst_cmd, SCST_CMD_DELIVERY_ABORTED);
req->scst_state = ISCSI_CMD_STATE_PROCESSED;
req_cmnd_release_force(req);
goto out;
}
TRACE_DBG("Prelim completed req %p", req);
/*
* We could preliminary have finished req before we
* knew its device, so check if we return correct sense
* format.
*/
scst_check_convert_sense(scst_cmd);
if (!req->own_sg) {
req->sg = scst_cmd_get_sg(scst_cmd);
req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
}
} else {
EXTRACHECKS_BUG_ON(req->own_sg);
req->sg = scst_cmd_get_sg(scst_cmd);
req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
}
req->bufflen = scst_cmd_get_adjusted_resp_data_len(scst_cmd);
req->scst_state = ISCSI_CMD_STATE_PROCESSED;
TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, req->sg_cnt %d",
req, is_send_status, req->bufflen, req->sg, req->sg_cnt);
EXTRACHECKS_BUG_ON(req->hashed);
if (req->main_rsp)
EXTRACHECKS_BUG_ON(cmnd_opcode(req->main_rsp) !=
ISCSI_OP_REJECT);
if (unlikely(req->bufflen != 0 && !is_send_status)) {
PRINT_CRIT_ERROR("Sending DATA without STATUS is unsupported");
scst_set_cmd_error(scst_cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
sBUG(); /* ToDo */
}
/*
* We need to decrement active_cmds before adding any responses into
* the write queue to eliminate a race, when all responses sent
* with wrong MaxCmdSN.
*/
if (likely(req->dec_active_cmds))
iscsi_dec_active_cmds(req);
if (req->bufflen != 0) {
/*
* Check above makes sure that is_send_status is set,
* so status is valid here, but in future that could change.
* ToDo
*/
req->conn->transport->iscsit_send_data_rsp(req, sense,
sense_len, status,
is_send_status);
} else if (is_send_status) {
struct iscsi_cmnd *rsp;
rsp = create_status_rsp(req, status, sense, sense_len);
iscsi_cmnd_init_write(rsp, 0);
}
#ifdef CONFIG_SCST_EXTRACHECKS
else
sBUG();
#endif
if (conn->transport->iscsit_send_locally(req, scst_get_active_cmd_count(scst_cmd)))
goto out_push_to_wr_thread;
out:
return SCST_TGT_RES_SUCCESS;
out_push_to_wr_thread:
TRACE_DBG("Waking up write thread (conn %p)", conn);
req_cmnd_release(req);
conn->transport->iscsit_make_conn_wr_active(conn);
goto out;
}
/* Called under sn_lock */
static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
{
bool res = 0;
struct iscsi_task_mgt_hdr *req_hdr =
(struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
int function = req_hdr->function & ISCSI_FUNCTION_MASK;
struct iscsi_session *sess = rsp->conn->session;
TRACE_ENTRY();
/* This should be checked for immediate TM commands as well */
#if 0
if (((scst_random() % 10) == 2))
res = 1;
#endif
switch (function) {
default:
if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
res = 1;
break;
}
TRACE_EXIT_RES(res);
return res;
}
/* Called under sn_lock, but might drop it inside, then reacquire */
static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
__acquires(&sn_lock)
__releases(&sn_lock)
{
struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
TRACE_ENTRY();
if (!tm_rsp)
goto out;
if (iscsi_is_delay_tm_resp(tm_rsp))
goto out;
TRACE_MGMT_DBG("Sending delayed rsp %p", tm_rsp);
sess->tm_rsp = NULL;
sess->tm_active--;
spin_unlock(&sess->sn_lock);
sBUG_ON(sess->tm_active < 0);
iscsi_cmnd_init_write(tm_rsp, ISCSI_INIT_WRITE_WAKE);
spin_lock(&sess->sn_lock);
out:
TRACE_EXIT();
}
static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status, bool drop)
{
struct iscsi_cmnd *rsp;
struct iscsi_task_mgt_hdr *req_hdr =
(struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
struct iscsi_task_rsp_hdr *rsp_hdr;
struct iscsi_session *sess = req->conn->session;
int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
TRACE_ENTRY();
TRACE_MGMT_DBG("TM req %p finished", req);
TRACE(TRACE_MGMT, "iSCSI TM fn %d finished, status %d, dropped %d",
fn, status, drop);
if (drop) {
spin_lock(&sess->sn_lock);
sess->tm_active--;
spin_unlock(&sess->sn_lock);
if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
struct iscsi_target *target = req->conn->session->target;
PRINT_INFO("Closing all connections for target %x at COLD RESET from initiator %s",
target->tid,
req->conn->session->initiator_name);
mutex_lock(&target->target_mutex);
target_del_all_sess(target, 0);
mutex_unlock(&target->target_mutex);
}
goto out_release;
}
rsp = iscsi_alloc_rsp(req);
rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
rsp_hdr->flags = ISCSI_FLG_FINAL;
rsp_hdr->itt = req_hdr->itt;
rsp_hdr->response = status;
if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
rsp->should_close_conn = 1;
rsp->should_close_all_conn = 1;
}
again:
spin_lock(&sess->sn_lock);
if (sess->tm_rsp) {
struct iscsi_cmnd *old_tm_rsp = sess->tm_rsp;
/*
* It is assumed that if we have another TM request, the
* original command the delayed TM response is waiting for
* is not going to come, hence we can drop it.
*/
iscsi_drop_delayed_tm_rsp(old_tm_rsp);
spin_unlock(&sess->sn_lock);
rsp_cmnd_release(old_tm_rsp);
goto again;
}
if (iscsi_is_delay_tm_resp(rsp)) {
TRACE_MGMT_DBG("Delaying TM fn %d response %p (req %p), because not all affected commands received (TM cmd sn %u, exp sn %u)",
req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
req_hdr->cmd_sn, sess->exp_cmd_sn);
sess->tm_rsp = rsp;
spin_unlock(&sess->sn_lock);
goto out_release;
}
sess->tm_active--;
sBUG_ON(sess->tm_active < 0);
spin_unlock(&sess->sn_lock);
iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
out_release:
req_cmnd_release(req);
TRACE_EXIT();
}
static inline int iscsi_get_mgmt_response(int status)
{
switch (status) {
case SCST_MGMT_STATUS_SUCCESS:
return ISCSI_RESPONSE_FUNCTION_COMPLETE;
case SCST_MGMT_STATUS_TASK_NOT_EXIST:
return ISCSI_RESPONSE_UNKNOWN_TASK;
case SCST_MGMT_STATUS_LUN_NOT_EXIST:
return ISCSI_RESPONSE_UNKNOWN_LUN;
case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
case SCST_MGMT_STATUS_REJECTED:
case SCST_MGMT_STATUS_FAILED:
default:
return ISCSI_RESPONSE_FUNCTION_REJECTED;
}
}
static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
{
int fn = scst_mgmt_cmd_get_fn(scst_mcmd);
struct iscsi_cmnd *req = scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
if (status == ISCSI_RESPONSE_UNKNOWN_TASK && fn == SCST_ABORT_TASK) {
/* If we are here, we found the task, so must succeed */
status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
}
TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d, status %d",
req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd),
status);
switch (fn) {
case SCST_NEXUS_LOSS_SESS:
/* Internal */
break;
case SCST_ABORT_ALL_TASKS_SESS:
case SCST_ABORT_ALL_TASKS:
case SCST_NEXUS_LOSS:
sBUG();
break;
default:
iscsi_send_task_mgmt_resp(req, status,
scst_mgmt_cmd_dropped(scst_mcmd));
scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
break;
}
}
static int iscsi_scsi_aen(struct scst_aen *aen)
{
int res = SCST_AEN_RES_SUCCESS;
__be64 lun = scst_aen_get_lun(aen);
const uint8_t *sense = scst_aen_get_sense(aen);
int sense_len = scst_aen_get_sense_len(aen);
struct iscsi_session *sess = scst_sess_get_tgt_priv(scst_aen_get_sess(aen));
struct iscsi_conn *conn;
bool found;
struct iscsi_cmnd *fake_req, *rsp;
struct iscsi_async_msg_hdr *rsp_hdr;
TRACE_ENTRY();
TRACE_MGMT_DBG("SCSI AEN to sess %p (initiator %s)",
sess, sess->initiator_name);
mutex_lock(&sess->target->target_mutex);
found = false;
list_for_each_entry_reverse(conn, &sess->conn_list, conn_list_entry) {
if (!test_bit(ISCSI_CONN_SHUTTINGDOWN, &conn->conn_aflags) &&
!conn->conn_reinst_successor) {
found = true;
break;
}
}
if (!found) {
TRACE_MGMT_DBG("Unable to find alive conn for sess %p", sess);
goto out_err_unlock;
}
/* Create a fake request */
fake_req = conn->transport->iscsit_alloc_cmd(conn, NULL);
if (!fake_req) {
PRINT_ERROR("Unable to alloc fake AEN request");
goto out_err_unlock;
}
mutex_unlock(&sess->target->target_mutex);
rsp = iscsi_alloc_main_rsp(fake_req);
if (!rsp) {
PRINT_ERROR("Unable to alloc AEN rsp");
goto out_err_free_req;
}
fake_req->scst_state = ISCSI_CMD_STATE_AEN;
fake_req->scst_aen = aen;
rsp_hdr = (struct iscsi_async_msg_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_ASYNC_MSG;
rsp_hdr->flags = ISCSI_FLG_FINAL;
rsp_hdr->lun = lun; /* it's already in SCSI form */
rsp_hdr->ffffffff = cpu_to_be32(0xffffffff);
rsp_hdr->async_event = ISCSI_ASYNC_SCSI;
rsp->sense_hdr.length = cpu_to_be16(sense_len);
rsp->conn->transport->iscsit_set_sense_data(rsp, sense, sense_len);
rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
rsp->bufflen = rsp->pdu.datasize;
req_cmnd_release(fake_req);
out:
TRACE_EXIT_RES(res);
return res;
out_err_free_req:
req_cmnd_release(fake_req);
goto out_set_res;
out_err_unlock:
mutex_unlock(&sess->target->target_mutex);
out_set_res:
res = SCST_AEN_RES_FAILED;
goto out;
}
static int iscsi_cpu_mask_changed_aen(struct scst_aen *aen)
{
int res = SCST_AEN_RES_SUCCESS;
struct scst_session *scst_sess = scst_aen_get_sess(aen);
struct iscsi_session *sess = scst_sess_get_tgt_priv(scst_sess);
TRACE_ENTRY();
TRACE_MGMT_DBG("CPU mask changed AEN to sess %p (initiator %s)",
sess, sess->initiator_name);
mutex_lock(&sess->target->target_mutex);
iscsi_sess_force_close(sess);
mutex_unlock(&sess->target->target_mutex);
scst_aen_done(aen);
TRACE_EXIT_RES(res);
return res;
}
static int iscsi_report_aen(struct scst_aen *aen)
{
int res;
int event_fn = scst_aen_get_event_fn(aen);
TRACE_ENTRY();
switch (event_fn) {
case SCST_AEN_SCSI:
res = iscsi_scsi_aen(aen);
break;
case SCST_AEN_CPU_MASK_CHANGED:
res = iscsi_cpu_mask_changed_aen(aen);
break;
default:
TRACE_MGMT_DBG("Unsupported AEN %d", event_fn);
res = SCST_AEN_RES_NOT_SUPPORTED;
break;
}
TRACE_EXIT_RES(res);
return res;
}
static int iscsi_get_initiator_port_transport_id(struct scst_tgt *tgt,
struct scst_session *scst_sess,
uint8_t **transport_id)
{
struct iscsi_session *sess;
int res = 0;
union iscsi_sid sid;
int tr_id_size;
uint8_t *tr_id;
TRACE_ENTRY();
if (!scst_sess) {
res = SCSI_TRANSPORTID_PROTOCOLID_ISCSI;
goto out;
}
sess = (struct iscsi_session *)scst_sess_get_tgt_priv(scst_sess);
sid = *(union iscsi_sid *)&sess->sid;
sid.id.tsih = 0;
tr_id_size = 4 + strlen(sess->initiator_name) + 5 +
snprintf(NULL, 0, "%llx", sid.id64) + 1;
tr_id_size = (tr_id_size + 3) & -4;
tr_id = kzalloc(tr_id_size, GFP_KERNEL);
if (!tr_id) {
PRINT_ERROR("Allocation of TransportID (size %d) failed",
tr_id_size);
res = -ENOMEM;
goto out;
}
tr_id[0] = 0x40 | SCSI_TRANSPORTID_PROTOCOLID_ISCSI;
sprintf(&tr_id[4], "%s,i,0x%llx", sess->initiator_name, sid.id64);
put_unaligned_be16(tr_id_size - 4, &tr_id[2]);
*transport_id = tr_id;
TRACE_DBG("Created tid '%s'", &tr_id[4]);
out:
TRACE_EXIT_RES(res);
return res;
}
void iscsi_send_nop_in(struct iscsi_conn *conn)
{
struct iscsi_cmnd *req, *rsp;
struct iscsi_nop_in_hdr *rsp_hdr;
TRACE_ENTRY();
req = conn->transport->iscsit_alloc_cmd(conn, NULL);
if (!req) {
PRINT_ERROR("Unable to alloc fake Nop-In request");
goto out_err;
}
rsp = iscsi_alloc_main_rsp(req);
if (!rsp) {
PRINT_ERROR("Unable to alloc Nop-In rsp");
goto out_err_free_req;
}
cmnd_get(rsp);
rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
rsp_hdr->opcode = ISCSI_OP_NOP_IN;
rsp_hdr->flags = ISCSI_FLG_FINAL;
rsp_hdr->itt = ISCSI_RESERVED_TAG;
rsp_hdr->ttt = (__force __be32)conn->nop_in_ttt++;
if (conn->nop_in_ttt == ISCSI_RESERVED_TAG_CPU32)
conn->nop_in_ttt = 0;
/* Supposed that all other fields are zeroed */
TRACE_DBG("Sending Nop-In request (ttt 0x%08x)", rsp_hdr->ttt);
spin_lock_bh(&conn->nop_req_list_lock);
list_add_tail(&rsp->nop_req_list_entry, &conn->nop_req_list);
spin_unlock_bh(&conn->nop_req_list_lock);
/*
* Start NopRsp timer now to catch case where send buffer is full due
* to connection being down, so this is never even sent to tcp layer.
* This prevent us from having to wait for the CmdRsp timer which
* is normally much longer.
*/
req_add_to_write_timeout_list(req);
out_err_free_req:
req_cmnd_release(req);
out_err:
TRACE_EXIT();
}
static int iscsi_close_sess(struct scst_session *scst_sess)
{
struct iscsi_session *sess = scst_sess_get_tgt_priv(scst_sess);
struct iscsi_target *target = sess->target;
int res;
res = mutex_lock_interruptible(&target->target_mutex);
if (res)
goto out;
iscsi_sess_force_close(sess);
mutex_unlock(&target->target_mutex);
res = 0;
out:
return res;
}
static int iscsi_target_release(struct scst_tgt *scst_tgt)
{
/* Nothing to do */
return 0;
}
#if !defined(CONFIG_SCST_PROC) && \
(defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING))
static struct scst_trace_log iscsi_local_trace_tbl[] = {
{ TRACE_D_WRITE, "d_write" },
{ TRACE_CONN_OC, "conn" },
{ TRACE_CONN_OC_DBG, "conn_dbg" },
{ TRACE_D_IOV, "iov" },
{ TRACE_D_DUMP_PDU, "pdu" },
{ TRACE_NET_PG, "net_page" },
{ 0, NULL }
};
#define ISCSI_TRACE_TBL_HELP ", d_write, conn, conn_dbg, iov, pdu, net_page"
#endif
static uint16_t iscsi_get_scsi_transport_version(struct scst_tgt *scst_tgt)
{
return 0x0960; /* iSCSI */
}
struct scst_tgt_template iscsi_template = {
.name = "iscsi",
.sg_tablesize = 0xFFFF /* no limit */,
.threads_num = 0,
.no_clustering = 1,
.xmit_response_atomic = 0,
.tgtt_attrs = iscsi_attrs,
.tgt_attrs = iscsi_tgt_attrs,
.sess_attrs = iscsi_sess_attrs,
.acg_attrs = iscsi_acg_attrs,
.enable_target = iscsi_enable_target,
.is_target_enabled = iscsi_is_target_enabled,
.add_target = iscsi_sysfs_add_target,
.del_target = iscsi_sysfs_del_target,
.mgmt_cmd = iscsi_sysfs_mgmt_cmd,
.close_session = iscsi_close_sess,
.tgtt_optional_attributes = "IncomingUser, OutgoingUser",
.tgt_optional_attributes = "IncomingUser, OutgoingUser, allowed_portal",
#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
.default_trace_flags = ISCSI_DEFAULT_LOG_FLAGS,
.trace_flags = &trace_flag,
#if !defined(CONFIG_SCST_PROC) && \
(defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING))
.trace_tbl = iscsi_local_trace_tbl,
.trace_tbl_help = ISCSI_TRACE_TBL_HELP,
#endif
#endif
.release = iscsi_target_release,
.xmit_response = iscsi_xmit_response,
.tgt_alloc_data_buf = iscsi_alloc_data_buf,
.preprocessing_done = iscsi_preprocessing_done,
.pre_exec = iscsi_pre_exec,
.task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done,
.task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
.on_abort_cmd = iscsi_on_abort_cmd,
.report_aen = iscsi_report_aen,
.get_initiator_port_transport_id =
iscsi_get_initiator_port_transport_id,
.get_scsi_transport_version = iscsi_get_scsi_transport_version,
};
static struct iscsit_transport iscsi_tcp_transport = {
.owner = THIS_MODULE,
.name = "iSCSI-TCP",
.transport_type = ISCSI_TCP,
.need_alloc_write_buf = 1,
.iscsit_conn_alloc = iscsi_conn_alloc,
.iscsit_conn_activate = conn_activate,
.iscsit_conn_free = iscsi_tcp_conn_free,
.iscsit_alloc_cmd = cmnd_alloc,
.iscsit_free_cmd = cmnd_free,
.iscsit_preprocessing_done = iscsi_tcp_preprocessing_done,
.iscsit_send_data_rsp = iscsi_tcp_send_data_rsp,
.iscsit_make_conn_wr_active = iscsi_make_conn_wr_active,
.iscsit_mark_conn_closed = iscsi_tcp_mark_conn_closed,
.iscsit_conn_close = iscsi_tcp_conn_close,
.iscsit_get_initiator_ip = iscsi_tcp_get_initiator_ip,
.iscsit_send_locally = iscsi_tcp_send_locally,
.iscsit_set_sense_data = iscsi_tcp_set_sense_data,
.iscsit_set_req_data = iscsi_tcp_set_req_data,
.iscsit_receive_cmnd_data = cmnd_rx_continue,
};
static void __iscsi_threads_pool_put(struct iscsi_thread_pool *p)
{
struct iscsi_thread *t, *tt;
TRACE_ENTRY();
p->thread_pool_ref--;
if (p->thread_pool_ref > 0) {
TRACE_DBG("iSCSI thread pool %p still has %d references)",
p, p->thread_pool_ref);
goto out;
}
TRACE_DBG("Freeing iSCSI thread pool %p", p);
mutex_lock(&p->tp_mutex);
list_for_each_entry_safe(t, tt, &p->threads_list, threads_list_entry) {
kthread_stop(t->thr);
list_del(&t->threads_list_entry);
kfree(t);
}
mutex_unlock(&p->tp_mutex);
list_del(&p->thread_pools_list_entry);
kmem_cache_free(iscsi_thread_pool_cache, p);
out:
TRACE_EXIT();
}
void iscsi_threads_pool_put(struct iscsi_thread_pool *p)
{
TRACE_ENTRY();
mutex_lock(&iscsi_threads_pool_mutex);
__iscsi_threads_pool_put(p);
mutex_unlock(&iscsi_threads_pool_mutex);
TRACE_EXIT();
}
int iscsi_threads_pool_get(bool dedicated, const cpumask_t *cpu_mask,
struct iscsi_thread_pool **out_pool)
{
int res;
struct iscsi_thread_pool *p;
struct iscsi_thread *t;
int i, j, count;
static int major; /* Protected by iscsi_threads_pool_mutex */
TRACE_ENTRY();
mutex_lock(&iscsi_threads_pool_mutex);
if (dedicated) {
/* Ignore cpu_mask, if set */
cpu_mask = NULL;
goto create;
}
list_for_each_entry(p, &iscsi_thread_pools_list, thread_pools_list_entry) {
if ((!cpu_mask || cpumask_equal(cpu_mask, &p->cpu_mask)) && !p->dedicated) {
p->thread_pool_ref++;
TRACE_DBG("iSCSI thread pool %p found (new ref %d)",
p, p->thread_pool_ref);
res = 0;
goto out_unlock;
}
}
create:
TRACE_DBG("Creating new iSCSI thread pool (dedicated %d, cpu_mask %p)",
dedicated, cpu_mask);
p = kmem_cache_zalloc(iscsi_thread_pool_cache, GFP_KERNEL);
if (!p) {
PRINT_ERROR("Unable to allocate iSCSI thread pool (size %zd)",
sizeof(*p));
res = 0;
if (!list_empty(&iscsi_thread_pools_list)) {
PRINT_WARNING("Using global iSCSI thread pool instead");
p = list_first_entry(&iscsi_thread_pools_list, struct iscsi_thread_pool,
thread_pools_list_entry);
} else {
res = -ENOMEM;
}
goto out_unlock;
}
spin_lock_init(&p->rd_lock);
INIT_LIST_HEAD(&p->rd_list);
init_waitqueue_head(&p->rd_waitQ);
spin_lock_init(&p->wr_lock);
INIT_LIST_HEAD(&p->wr_list);
init_waitqueue_head(&p->wr_waitQ);
if (!cpu_mask)
cpumask_setall(&p->cpu_mask);
else
cpumask_copy(&p->cpu_mask, cpu_mask);
p->thread_pool_ref = 1;
mutex_init(&p->tp_mutex);
INIT_LIST_HEAD(&p->threads_list);
p->dedicated = dedicated;
if (dedicated) {
count = 1;
} else if (!cpu_mask) {
count = max_t(int, blk_mq_num_online_queues(0), 2);
} else {
count = 0;
for_each_cpu(i, cpu_mask)
count++;
}
list_add_tail(&p->thread_pools_list_entry, &iscsi_thread_pools_list);
for (j = 0; j < 2; j++) {
for (i = 0; i < count; i++) {
t = kmalloc_obj(*t);
if (!t) {
res = -ENOMEM;
PRINT_ERROR("Failed to allocate thread (size %zd)",
sizeof(*t));
goto out_free;
}
t->thr = kthread_run(j ? istwr : istrd, p,
"iscsi%s%d_%d", j ? "wr" : "rd",
major, i);
if (IS_ERR(t->thr)) {
res = PTR_ERR(t->thr);
PRINT_ERROR("kthread_run() failed: %d", res);
kfree(t);
goto out_free;
}
mutex_lock(&p->tp_mutex);
list_add_tail(&t->threads_list_entry, &p->threads_list);
mutex_unlock(&p->tp_mutex);
}
}
major++;
res = 0;
TRACE_DBG("Created iSCSI thread pool %p", p);
out_unlock:
mutex_unlock(&iscsi_threads_pool_mutex);
*out_pool = p;
TRACE_EXIT_RES(res);
return res;
out_free:
__iscsi_threads_pool_put(p);
p = NULL;
goto out_unlock;
}
static int __init iscsi_init(void)
{
int err = 0;
PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
err = iscsit_reg_transport(&iscsi_tcp_transport);
if (err)
goto out;
dummy_page = alloc_pages(GFP_KERNEL, 0);
if (!dummy_page) {
PRINT_ERROR("Dummy page allocation failed");
goto out;
}
sg_init_table(&dummy_sg[0], ARRAY_SIZE(dummy_sg));
sg_set_page(&dummy_sg[0], dummy_page, PAGE_SIZE, 0);
iscsi_cmnd_abort_mempool = mempool_create_kmalloc_pool(2500,
sizeof(struct iscsi_cmnd_abort_params));
if (!iscsi_cmnd_abort_mempool) {
err = -ENOMEM;
goto out_free_dummy;
}
ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
if (ctr_major < 0) {
PRINT_ERROR("failed to register the control device %d",
ctr_major);
err = ctr_major;
goto out_callb;
}
err = event_init();
if (err < 0)
goto out_reg;
iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd,
SCST_SLAB_FLAGS | SLAB_HWCACHE_ALIGN);
if (!iscsi_cmnd_cache) {
err = -ENOMEM;
goto out_event;
}
iscsi_thread_pool_cache = KMEM_CACHE(iscsi_thread_pool,
SCST_SLAB_FLAGS | SLAB_HWCACHE_ALIGN);
if (!iscsi_thread_pool_cache) {
err = -ENOMEM;
goto out_kmem_cmd;
}
iscsi_conn_cache = KMEM_CACHE(iscsi_conn,
SCST_SLAB_FLAGS | SLAB_HWCACHE_ALIGN);
if (!iscsi_conn_cache) {
err = -ENOMEM;
goto out_kmem_tp;
}
iscsi_sess_cache = KMEM_CACHE(iscsi_session, SCST_SLAB_FLAGS | SLAB_HWCACHE_ALIGN);
if (!iscsi_sess_cache) {
err = -ENOMEM;
goto out_kmem_conn;
}
err = scst_register_target_template(&iscsi_template);
if (err < 0)
goto out_kmem;
iscsi_conn_ktype.sysfs_ops = scst_sysfs_get_sysfs_ops();
err = iscsi_threads_pool_get(false, NULL, &iscsi_main_thread_pool);
if (err != 0)
goto out_thr;
out:
return err;
out_thr:
scst_unregister_target_template(&iscsi_template);
out_kmem:
kmem_cache_destroy(iscsi_sess_cache);
out_kmem_conn:
kmem_cache_destroy(iscsi_conn_cache);
out_kmem_tp:
kmem_cache_destroy(iscsi_thread_pool_cache);
out_kmem_cmd:
kmem_cache_destroy(iscsi_cmnd_cache);
out_event:
event_exit();
out_reg:
unregister_chrdev(ctr_major, ctr_name);
out_callb:
mempool_destroy(iscsi_cmnd_abort_mempool);
out_free_dummy:
__free_pages(dummy_page, 0);
goto out;
}
static void __exit iscsi_exit(void)
{
iscsi_threads_pool_put(iscsi_main_thread_pool);
sBUG_ON(!list_empty(&iscsi_thread_pools_list));
unregister_chrdev(ctr_major, ctr_name);
event_exit();
kmem_cache_destroy(iscsi_sess_cache);
kmem_cache_destroy(iscsi_conn_cache);
kmem_cache_destroy(iscsi_thread_pool_cache);
kmem_cache_destroy(iscsi_cmnd_cache);
scst_unregister_target_template(&iscsi_template);
iscsit_unreg_transport(&iscsi_tcp_transport);
mempool_destroy(iscsi_cmnd_abort_mempool);
__free_pages(dummy_page, 0);
}
module_init(iscsi_init);
module_exit(iscsi_exit);
MODULE_VERSION(ISCSI_VERSION_STRING);
MODULE_LICENSE("GPL");
MODULE_IMPORT_NS(SCST_NAMESPACE);
MODULE_DESCRIPTION("SCST iSCSI Target");
Reference in New Issue View Git Blame Copy Permalink