From d51ecb0e2a469644d3cdf17f8a4ef68a300f1664 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 27 Nov 2014 15:19:21 +0000 Subject: [PATCH] ib_srpt: Add support for immediate data git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@5892 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- srpt/src/ib_srpt.c | 132 ++++++++++++++++++++++++++++++++++++--------- srpt/src/ib_srpt.h | 24 ++++++--- srpt/src/srp-ext.h | 22 ++++++++ 3 files changed, 147 insertions(+), 31 deletions(-) create mode 100644 srpt/src/srp-ext.h diff --git a/srpt/src/ib_srpt.c b/srpt/src/ib_srpt.c index 6c12d758e..2537d688f 100644 --- a/srpt/src/ib_srpt.c +++ b/srpt/src/ib_srpt.c @@ -50,6 +50,7 @@ #endif #endif #include "ib_srpt.h" +#include "srp-ext.h" #define LOG_PREFIX "ib_srpt" /* Prefix for SCST tracing macros. */ #if defined(INSIDE_KERNEL_TREE) #include @@ -789,21 +790,27 @@ static void srpt_unregister_mad_agent(struct srpt_device *sdev) */ static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev, int ioctx_size, int dma_size, + int alignment_offset, enum dma_data_direction dir) { struct srpt_ioctx *ioctx; - ioctx = kmalloc(ioctx_size, GFP_KERNEL); + ioctx = kzalloc(ioctx_size, GFP_KERNEL); if (!ioctx) goto err; - ioctx->buf = kmalloc(dma_size, GFP_KERNEL); + ioctx->buf = kmalloc(dma_size + alignment_offset, GFP_KERNEL); if (!ioctx->buf) goto err_free_ioctx; - ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir); + /* Complain if it is not safe to use zero-copy */ + WARN_ON_ONCE(alignment_offset && ((uintptr_t)ioctx->buf & 511)); + + ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, + dma_size + alignment_offset, dir); if (ib_dma_mapping_error(sdev->device, ioctx->dma)) goto err_free_buf; + ioctx->offset = alignment_offset; return ioctx; @@ -839,7 +846,8 @@ static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx, */ static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev, int ring_size, int ioctx_size, - int dma_size, enum dma_data_direction dir) + int dma_size, int alignment_offset, + enum dma_data_direction dir) { struct srpt_ioctx **ring; int i; @@ -853,7 +861,8 @@ static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev, if (!ring) goto out; for (i = 0; i < ring_size; ++i) { - ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir); + ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, + alignment_offset, dir); if (!ring[i]) goto err; ring[i]->index = i; @@ -862,7 +871,7 @@ static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev, err: while (--i >= 0) - srpt_free_ioctx(sdev, ring[i], dma_size, dir); + srpt_free_ioctx(sdev, ring[i], dma_size + ring[i]->offset, dir); kfree(ring); ring = NULL; out: @@ -883,7 +892,8 @@ static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring, return; for (i = 0; i < ring_size; ++i) - srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir); + srpt_free_ioctx(sdev, ioctx_ring[i], + dma_size + ioctx_ring[i]->offset, dir); kfree(ioctx_ring); } @@ -943,7 +953,7 @@ static int srpt_post_recv(struct srpt_device *sdev, struct srpt_rdma_ch *ch, BUG_ON(!sdev); wr.wr_id = encode_wr_id(SRPT_RECV, ioctx->ioctx.index); - list.addr = ioctx->ioctx.dma; + list.addr = ioctx->ioctx.dma + ioctx->ioctx.offset; list.length = srp_max_req_size; list.lkey = sdev->mr->lkey; @@ -1038,7 +1048,8 @@ static int srpt_zerolength_write(struct srpt_rdma_ch *ch) * Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors; * -ENOMEM when memory allocation fails and zero upon success. */ -static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx, +static int srpt_get_desc_tbl(struct srpt_recv_ioctx *recv_ioctx, + struct srpt_send_ioctx *ioctx, struct srp_cmd *srp_cmd, scst_data_direction *dir, u64 *data_len) { @@ -1090,7 +1101,38 @@ static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx, * is four times the value specified in bits 3..7. Hence the "& ~3". */ add_cdb_offset = srp_cmd->add_cdb_len & ~3; - if (fmt == SRP_DATA_DESC_DIRECT) { + if (fmt == SRP_DATA_DESC_IMM) { + struct srp_imm_buf *imm_buf = (void *)(srp_cmd->add_data + + add_cdb_offset); + void *data; + uint32_t header_size; + uint64_t req_size; + + header_size = be32_to_cpu(imm_buf->offset); + *data_len = be32_to_cpu(imm_buf->len); + req_size = header_size + *data_len; + data = (void *)srp_cmd + header_size; + if (req_size > srp_max_req_size) { + PRINT_ERROR("Immediate data (length %d + %lld) exceeds" + " request size %d", header_size, *data_len, + srp_max_req_size); + ret = -EINVAL; + goto out; + } + if (WARN_ONCE(recv_ioctx->byte_len < req_size, + "received too few data - %d < %lld\n", + recv_ioctx->byte_len, req_size)) { + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, + 1, srp_cmd, recv_ioctx->byte_len, 1); + ret = -EIO; + } + ioctx->imm_data = data; + ioctx->recv_ioctx = recv_ioctx; + if (((uintptr_t)data & 511) == 0) { + sg_init_one(&ioctx->imm_sg, ioctx->imm_data, *data_len); + scst_cmd_set_tgt_sg(&ioctx->scmnd, &ioctx->imm_sg, 1); + } + } else if (fmt == SRP_DATA_DESC_DIRECT) { ioctx->n_rbuf = 1; ioctx->rbufs = &ioctx->single_rbuf; @@ -1286,8 +1328,10 @@ static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch) BUG_ON(ioctx->ch != ch); spin_lock_init(&ioctx->spinlock); ioctx->state = SRPT_STATE_NEW; + EXTRACHECKS_WARN_ON(ioctx->recv_ioctx); ioctx->n_rbuf = 0; ioctx->rbufs = NULL; + ioctx->imm_data = NULL; ioctx->n_rdma = 0; ioctx->n_rdma_ius = 0; ioctx->rdma_ius = NULL; @@ -1302,12 +1346,15 @@ static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch) */ static void srpt_put_send_ioctx(struct srpt_send_ioctx *ioctx) { - struct srpt_rdma_ch *ch; + struct srpt_rdma_ch *ch = ioctx->ch; + struct srpt_recv_ioctx *recv_ioctx = ioctx->recv_ioctx; unsigned long flags; - BUG_ON(!ioctx); - ch = ioctx->ch; - BUG_ON(!ch); + if (recv_ioctx) { + EXTRACHECKS_WARN_ON(!list_empty(&recv_ioctx->wait_list)); + ioctx->recv_ioctx = NULL; + srpt_post_recv(ch->sport->sdev, ch, recv_ioctx); + } /* * If the WARN_ON() below gets triggered this means that @@ -1687,7 +1734,7 @@ static int srpt_handle_cmd(struct srpt_rdma_ch *ch, BUG_ON(!send_ioctx); - srp_cmd = recv_ioctx->ioctx.buf; + srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset; scmnd = &send_ioctx->scmnd; ret = scst_rx_cmd_prealloced(scmnd, ch->scst_sess, (u8 *) &srp_cmd->lun, @@ -1699,7 +1746,8 @@ static int srpt_handle_cmd(struct srpt_rdma_ch *ch, goto err; } - ret = srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &data_len); + ret = srpt_get_desc_tbl(recv_ioctx, send_ioctx, srp_cmd, &dir, + &data_len); if (ret) { PRINT_ERROR("0x%llx: parsing SRP descriptor table failed.", srp_cmd->tag); @@ -1765,7 +1813,7 @@ static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch, srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT); - srp_tsk = recv_ioctx->ioctx.buf; + srp_tsk = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset; TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld" " using tag= %lld ch= %p sess= %p", @@ -1855,10 +1903,11 @@ srpt_handle_new_iu(struct srpt_rdma_ch *ch, goto push; ib_dma_sync_single_for_cpu(ch->sport->sdev->device, - recv_ioctx->ioctx.dma, srp_max_req_size, + recv_ioctx->ioctx.dma, + recv_ioctx->ioctx.offset + srp_max_req_size, DMA_FROM_DEVICE); - srp_cmd = recv_ioctx->ioctx.buf; + srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset; opcode = srp_cmd->opcode; if (opcode == SRP_CMD || opcode == SRP_TSK_MGMT) { send_ioctx = srpt_get_send_ioctx(ch); @@ -1893,7 +1942,8 @@ srpt_handle_new_iu(struct srpt_rdma_ch *ch, break; } - srpt_post_recv(ch->sport->sdev, ch, recv_ioctx); + if (!send_ioctx || !send_ioctx->recv_ioctx) + srpt_post_recv(ch->sport->sdev, ch, recv_ioctx); out: return send_ioctx; @@ -1923,7 +1973,7 @@ static void srpt_process_rcv_completion(struct ib_cq *cq, ioctx = ch->sport->sdev->ioctx_ring[index]; else ioctx = ch->ioctx_recv_ring[index]; - + ioctx->byte_len = wc->byte_len; srpt_handle_new_iu(ch, ioctx, srpt_new_iu_context); } else { PRINT_INFO("receiving failed for idx %u with status %d", @@ -2189,6 +2239,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) qp_init->cap.max_send_wr = srpt_sq_size; ch->max_sge = max_t(int, 1, sdev->dev_attr.max_sge - max_sge_delta); qp_init->cap.max_send_sge = ch->max_sge; + qp_init->cap.max_recv_sge = ch->max_sge; if (sdev->use_srq) { qp_init->srq = sdev->srq; } else { @@ -2610,7 +2661,7 @@ static int srpt_cm_req_recv(struct srpt_device *const sdev, ch->ioctx_ring = (struct srpt_send_ioctx **) srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size, sizeof(*ch->ioctx_ring[0]), - ch->max_rsp_size, DMA_TO_DEVICE); + ch->max_rsp_size, 0, DMA_TO_DEVICE); if (!ch->ioctx_ring) { PRINT_ERROR("rejected SRP_LOGIN_REQ because creating" " a new QP SQ ring failed."); @@ -2628,6 +2679,7 @@ static int srpt_cm_req_recv(struct srpt_device *const sdev, srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size, sizeof(*ch->ioctx_recv_ring[0]), srp_max_req_size, + DATA_ALIGNMENT_OFFSET, DMA_FROM_DEVICE); if (!ch->ioctx_recv_ring) { PRINT_ERROR("rejected SRP_LOGIN_REQ because creating" @@ -2743,11 +2795,12 @@ static int srpt_cm_req_recv(struct srpt_device *const sdev, /* create srp_login_response */ rsp->opcode = SRP_LOGIN_RSP; rsp->tag = req->tag; - rsp->max_it_iu_len = req->req_it_iu_len; + rsp->max_it_iu_len = cpu_to_be32(srp_max_req_size); rsp->max_ti_iu_len = req->req_it_iu_len; ch->max_ti_iu_len = it_iu_len; rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT | - SRP_BUF_FORMAT_INDIRECT); + SRP_BUF_FORMAT_INDIRECT | + SRP_BUF_FORMAT_IMM); rsp->req_lim_delta = cpu_to_be32(ch->rq_size); ch->req_lim = ch->rq_size; ch->req_lim_delta = 0; @@ -3331,6 +3384,10 @@ static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch, EXTRACHECKS_BUG_ON(!ch); EXTRACHECKS_BUG_ON(!ioctx); + + if (ioctx->imm_data) + return; + EXTRACHECKS_BUG_ON(ioctx->n_rdma && !ioctx->rdma_ius); if (ioctx->rdma_ius != (void *)ioctx->rdma_ius_buf) @@ -3456,6 +3513,30 @@ static int srpt_xfer_data(struct srpt_rdma_ch *ch, { int ret; + if (ioctx->imm_data) { + BUG_ON(!srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA, + SRPT_STATE_DATA_IN)); + if (unlikely(!scst_cmd_get_tgt_data_buff_alloced(scmnd))) { + unsigned offset = 0, len; + uint8_t *buf; + + len = scst_get_buf_first(scmnd, &buf); + while (len > 0) { + memcpy(buf, ioctx->imm_data + offset, len); + offset += len; + len = scst_get_buf_next(scmnd, &buf); + } + WARN_ON_ONCE(offset != + scst_cmd_get_expected_transfer_len(scmnd)); + } + scst_rx_data(scmnd, SCST_RX_STATUS_SUCCESS, + in_irq() ? SCST_CONTEXT_TASKLET : + in_softirq() ? SCST_CONTEXT_DIRECT_ATOMIC : + SCST_CONTEXT_DIRECT); + ret = SCST_TGT_RES_SUCCESS; + goto out; + } + ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd); if (ret) { PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret); @@ -4257,6 +4338,7 @@ static void srpt_add_one(struct ib_device *device) srpt_alloc_ioctx_ring(sdev, sdev->srq_size, sizeof(*sdev->ioctx_ring[0]), srp_max_req_size, + DATA_ALIGNMENT_OFFSET, DMA_FROM_DEVICE); if (!sdev->ioctx_ring) { PRINT_ERROR("srpt_alloc_ioctx_ring() failed"); @@ -4503,6 +4585,8 @@ static int __init srpt_init_module(void) { int ret; + BUILD_BUG_ON(sizeof(struct srp_imm_buf) != 8); + ret = -EINVAL; if (srp_max_req_size < MIN_MAX_REQ_SIZE) { PRINT_ERROR("invalid value %d for kernel module parameter" diff --git a/srpt/src/ib_srpt.h b/srpt/src/ib_srpt.h index f62aabaf4..30abefbeb 100644 --- a/srpt/src/ib_srpt.h +++ b/srpt/src/ib_srpt.h @@ -56,6 +56,7 @@ #endif #include #include +#include "srp-ext.h" #include "ib_dm_mad.h" /* @@ -128,10 +129,9 @@ enum { MAX_SRPT_SRQ_SIZE = 65535, MIN_MAX_REQ_SIZE = 996, - DEFAULT_MAX_REQ_SIZE - = sizeof(struct srp_cmd)/*48*/ - + sizeof(struct srp_indirect_buf)/*20*/ - + 255 * sizeof(struct srp_direct_buf)/*16*/, + SRP_IMM_DATA_OUT_OFFSET = 80, + DEFAULT_MAX_REQ_SIZE = SRP_IMM_DATA_OUT_OFFSET + 8192, + DATA_ALIGNMENT_OFFSET = 512 - SRP_IMM_DATA_OUT_OFFSET, MIN_MAX_RSP_SIZE = sizeof(struct srp_rsp)/*36*/ + 4, DEFAULT_MAX_RSP_SIZE = 256, /* leaves 220 bytes for sense data */ @@ -207,13 +207,15 @@ enum srpt_command_state { /** * struct srpt_ioctx - Shared SRPT I/O context information. - * @buf: Pointer to the buffer. - * @dma: DMA address of the buffer. - * @index: Index of the I/O context in its ioctx_ring array. + * @buf: Pointer to the buffer. + * @dma: DMA address of the buffer. + * @offset: Offset of the first byte in @buf and @dma that is actually used. + * @index: Index of the I/O context in its ioctx_ring array. */ struct srpt_ioctx { void *buf; dma_addr_t dma; + uint32_t offset; uint32_t index; }; @@ -221,10 +223,12 @@ struct srpt_ioctx { * struct srpt_recv_ioctx - SRPT receive I/O context. * @ioctx: See above. * @wait_list: Node for insertion in srpt_rdma_ch.cmd_wait_list. + * @byte_len: Number of bytes in @ioctx.buf. */ struct srpt_recv_ioctx { struct srpt_ioctx ioctx; struct list_head wait_list; + int byte_len; }; /** @@ -239,7 +243,10 @@ struct srpt_tsk_mgmt { * struct srpt_send_ioctx - SRPT send I/O context. * @ioctx: See above. * @ch: Channel pointer. + * @recv_ioctx: Receive I/O context associated with this send I/O context. * @rdma_ius: Array with information about the RDMA mapping. + * @imm_data: Pointer to immediate data when using the immediate data format. + * @imm_sg: Scatterlist for immediate data. * @rbufs: Pointer to SRP data buffer array. * @single_rbuf: SRP data buffer if the command has only a single buffer. * @sg: Pointer to sg-list associated with this I/O context. @@ -263,7 +270,10 @@ struct srpt_tsk_mgmt { struct srpt_send_ioctx { struct srpt_ioctx ioctx; struct srpt_rdma_ch *ch; + struct srpt_recv_ioctx *recv_ioctx; struct rdma_iu *rdma_ius; + void *imm_data; + struct scatterlist imm_sg; struct srp_direct_buf *rbufs; struct srp_direct_buf single_rbuf; struct scatterlist *sg; diff --git a/srpt/src/srp-ext.h b/srpt/src/srp-ext.h new file mode 100644 index 000000000..3e6752cec --- /dev/null +++ b/srpt/src/srp-ext.h @@ -0,0 +1,22 @@ +/* + * Extensions to the SRPr16a protocol + * + * Copyright (C) 2013 Fusion-io, Inc. All rights reserved. + */ + +#ifndef _SRP_EXT_H_ +#define _SRP_EXT_H_ + +/* + * Data is present as immediate data instead of being referred to via a + * descriptor. + */ +enum { SRP_DATA_DESC_IMM = 3 }; +enum { SRP_BUF_FORMAT_IMM = 1 << 3 }; + +struct srp_imm_buf { + __be32 len; + __be32 offset; +}; + +#endif /* _SRP_EXT_H_ */