From bef07c9d31713053098b32748a9337111977dfee Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Mon, 18 Jun 2012 12:10:53 +0000 Subject: [PATCH] ib_srpt: Add one target per port support. Makes it possible to configure LUN masking per IB HCA port instead of per HCA, just like what is possible with FC. git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@4354 d57e44dd-8a1f-0410-8b47-8ef2f437770f --- srpt/README | 113 +++++++------- srpt/src/ib_srpt.c | 364 +++++++++++++++++++++++++++++---------------- srpt/src/ib_srpt.h | 48 +++--- 3 files changed, 324 insertions(+), 201 deletions(-) diff --git a/srpt/README b/srpt/README index 961b35fe9..240781fc1 100644 --- a/srpt/README +++ b/srpt/README @@ -35,6 +35,19 @@ Building and installing the SRP target driver is possible as follows: make -s scstadm scstadm_install The ib_srpt kernel module supports the following parameters: +* one_target_per_port (boolean) and +* use_node_guid_in_target_name (boolean) + ib_srpt can operate in one of the following three modes: + 1. Access control configuration per HCA and assigning a "ib_srpt_target_" + style name to each HCA. + 2. Access control configuration per HCA and referring to a HCA via its node + GUID (e.g. 0002:c903:0005:f34a). + 3. Access control configuration per HCA port and referring to a HCA via its + port GID (e.g. fe80:0000:0000:0000:0002:c903:0005:f34b). + Mode (1) is choosen if both one_target_per_port and + use_node_guid_in_target_name are false. Mode (2) is choosen if + one_target_per_port is false and use_node_guid_in_target_name is true. Mode + (3) is choosen if one_target_per_port is true. * srp_max_req_size (number) Maximum size of an SRP control message in bytes. Examples of SRP control messages are: login request, logout request, data transfer request, ... @@ -68,7 +81,14 @@ The ib_srpt kernel module supports the following parameters: Configuring the SRP Target System --------------------------------- -First of all, create the file /etc/scst.conf. You can create this file with +The first step is to choose whether access control will be controlled per +HCA or per HCA port and to create a modprobe configuration file that reflects +this choice. An example: + + # cat /etc/modprobe.d/ib_srpt.conf + options ib_srpt one_target_per_port=1 + +Next, create the file /etc/scst.conf. You can create this file with the scstadmin tool as follows: /etc/init.d/scst stop @@ -121,15 +141,15 @@ The meaning of the parameters in the above command is as follows: Target GIDs can be queried e.g. via sysfs: $ for f in /sys/devices/*/*/*/infiniband/*/ports/*/gids/0; do echo $f; \ -cat $f | cut -c21- | sed 's/://g'; done +cat $f | sed 's/://g'; done /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/infiniband/mlx4_0/ports/1/gids/0 -0002c9030005f34b +fe800000000000000002c9030005f34b /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/infiniband/mlx4_0/ports/2/gids/0 -0002c9030005f34c +fe800000000000000002c9030005f34c /sys/devices/pci0000:00/0000:00:1c.0/0000:05:00.0/infiniband/mlx4_1/ports/1/gids/0 -0002c9030003cca7 +fe800000000000000002c9030003cca7 /sys/devices/pci0000:00/0000:00:1c.0/0000:05:00.0/infiniband/mlx4_1/ports/2/gids/0 -0002c9030003cca8 +fe800000000000000002c9030003cca8 Finally run lsscsi to display the details of the newly discovered SCSI disks: @@ -144,46 +164,38 @@ the disk names assigned on the SCST target ("disk01" in the example below): Target names ------------ -The name assigned by the ib_srpt target driver to an SCST target is the node -GUID of a HCA in hexadecimal form with a colon after every fourth digit. The -HCA node GUIDs can be obtained via the ibv_devices command, the ibv_devinfo -command or via sysfs. An example: +The name assigned by the ib_srpt target driver to an SCST target is either +ib_srpt_target_, the node GUID of a HCA in hexadecimal form with a colon +after every fourth digit or the port GUID with a colon afer every fourth +digit. The HCA node and port GUIDs can be obtained via the ibv_devinfo +command. An example: -# ibv_devices - device node GUID - ------ ---------------- - mlx4_1 0002c9030003cca2 - mlx4_0 0002c9030005f34e +# ibv_devinfo -v | grep -E '[^a-z]port:|guid|GID' +node_guid: 0002:c903:0005:f34e +sys_image_guid: 0002:c903:0005:f351 + port: 1 + GID[0]: fe80:0000:0000:0000:0002:c903:0005:f34f + port: 2 + GID[0]: fe80:0000:0000:0000:0002:c903:0005:f350 -# head /sys/devices/*/*/*/infiniband/*/node_guid -==> /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/infiniband/mlx4_0/node_guid <== -0002:c903:0005:f34e +Once the ib_srpt driver has been loaded the available SCST targets can be +queried as follows: -==> /sys/devices/pci0000:00/0000:00:1c.0/0000:05:00.0/infiniband/mlx4_1/node_guid <== -0002:c903:0003:cca2 - -Once the ib_srpt driver has been loaded there will be SCST targets available -with the HCA node GUID as name: - -# ls /sys/bus/scst_target/drivers/ib_srpt/0* -0002:c903:0003:cca2 0002:c903:0005:f34e - -If you need deprecated target names in form ib_srpt_target_X, you should -set use_node_guid_in_target_name parameter of module ib_srpt to 0. - -To move from the deprecated ib_srpt_target_X layout you should replace -in your scst.conf all ib_srpt_target_X to the corresponding ports names, -like ib_srpt_target_0 to 0002:c902:0022:16f4, if ibv_devices reported -for this port/node GUID 0002c902002216f4. +# (cd /sys/kernel/scst_tgt/targets/ib_srpt && ls -d [0-9a-f]*) +fe80:0000:0000:0000:0002:c903:0005:f34f +fe80:0000:0000:0000:0002:c903:0005:f350 Session names ------------- -The ib_srpt target driver uses the 128-bit SRP initiator port identifier for -the session name. This identifier is sent by the SRP initiator to the SRP -target via the SRP_LOGIN_REQ information unit. The Linux SRP initiator -(ib_srp) generates the initiator port identifier as follows: +The name assigned by the ib_srpt target driver to a session depends on the +mode in which it is operating. If one_target_per_port=y then the source port +GID is used as the session name. If one_target_per_port=n then the 128-bit SRP +initiator port identifier is used as the session name. This identifier is sent +by the SRP initiator to the SRP target via the SRP_LOGIN_REQ information unit. +The Linux SRP initiator (ib_srp) generates the initiator port identifier as +follows: - The first eight bytes are the identifier extension ('initiator_ext' parameter specified in the login string echoed into the sysfs file 'add_target'). - The last eight bytes are the GUID of the initiator HCA port used to @@ -194,24 +206,21 @@ An example: [ INITIATOR ] $ for f in /sys/devices/*/*/*/infiniband/*/ports/*/gids/0; do echo -f; cat $f | cut -c21-; done +f; cat $f; done /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/infiniband/mlx4_0/ports/1/gids/0 -0002:c903:0005:f34b +fe80:0000:0000:0000:0002:c903:0005:f34b /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/infiniband/mlx4_0/ports/2/gids/0 -0002:c903:0005:f34c +fe80:0000:0000:0000:0002:c903:0005:f34c /sys/devices/pci0000:00/0000:00:1c.0/0000:05:00.0/infiniband/mlx4_1/ports/1/gids/0 -0002:c903:0003:cca7 +fe80:0000:0000:0000:0002:c903:0003:cca7 /sys/devices/pci0000:00/0000:00:1c.0/0000:05:00.0/infiniband/mlx4_1/ports/2/gids/0 -0002:c903:0003:cca8 +fe80:0000:0000:0000:0002:c903:0003:cca8 [ TARGET, after login ] -$ ls /sys/bus/scst_target/drivers/ib_srpt/*/sessions -/sys/bus/scst_target/drivers/ib_srpt/0002:c903:0003:cca2/sessions: -0x00000000000000000002c9030003cca7 - -/sys/bus/scst_target/drivers/ib_srpt/0002:c903:0005:f34e/sessions: -0x00000000000000000002c9030005f34b +$ (cd /sys/kernel/scst_tgt/targets/ib_srpt/[0-9a-f]* && ls -d sessions/*) +sessions/fe80:0000:0000:0000:0002:c903:0003:cca7 +sessions/fe80:0000:0000:0000:0002:c903:0005:f34b LUN masking @@ -224,7 +233,7 @@ identified by their session name (see above). An example of an scst.conf file using LUN masking for ib_srpt: TARGET_DRIVER ib_srpt { - TARGET ib_srpt_target_0 { + TARGET fe80:0000:0000:0000:0002:c903:0005:f34b { enabled 1 rel_tgt_id 1 @@ -235,14 +244,14 @@ TARGET_DRIVER ib_srpt { # LUNs visible by initiator system 1 LUN 0 disk02 - INITIATOR 0x00000000000000000002c9030005f34b + INITIATOR fe80:0000:0000:0000:0002:c903:0005:f34b } GROUP grp2 { # LUNs visible by initiator system 2 LUN 0 disk03 - INITIATOR 0x00000000000000000002c9030005f34c + INITIATOR fe80:0000:0000:0000:0002:c903:0005:f34c } } } diff --git a/srpt/src/ib_srpt.c b/srpt/src/ib_srpt.c index d50de4491..2015ae220 100644 --- a/srpt/src/ib_srpt.c +++ b/srpt/src/ib_srpt.c @@ -132,7 +132,17 @@ static bool use_node_guid_in_target_name; #endif module_param(use_node_guid_in_target_name, bool, 0444); MODULE_PARM_DESC(use_node_guid_in_target_name, - "Use target node GUIDs of HCAs as SCST target names."); + "Use HCA node GUID as SCST target name."); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31) \ + || defined(RHEL_MAJOR) && RHEL_MAJOR -0 <= 5 +static int one_target_per_port; +#else +static bool one_target_per_port; +#endif +module_param(one_target_per_port, bool, 0444); +MODULE_PARM_DESC(one_target_per_port, + "One SCST target per HCA port instead of one per HCA."); static int srpt_get_u64_x(char *buffer, struct kernel_param *kp) { @@ -154,6 +164,7 @@ static const enum scst_exec_context srpt_xmt_rsp_context = SCST_CONTEXT_THREAD; static const enum scst_exec_context srpt_send_context = SCST_CONTEXT_DIRECT; static struct ib_client srpt_client; +static struct scst_tgt_template srpt_template; static void srpt_unregister_mad_agent(struct srpt_device *sdev); #ifdef CONFIG_SCST_PROC static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt); @@ -678,6 +689,7 @@ static int srpt_refresh_port(struct srpt_port *sport) struct ib_port_modify port_modify; struct ib_port_attr port_attr; int ret; + char tgt_name[40]; TRACE_ENTRY(); @@ -721,6 +733,26 @@ static int srpt_refresh_port(struct srpt_port *sport) } } + if (one_target_per_port && !sport->srpt_tgt.scst_tgt) { + snprintf(tgt_name, sizeof(tgt_name), + "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x", + be16_to_cpu(((__be16 *) sport->gid.raw)[0]), + be16_to_cpu(((__be16 *) sport->gid.raw)[1]), + be16_to_cpu(((__be16 *) sport->gid.raw)[2]), + be16_to_cpu(((__be16 *) sport->gid.raw)[3]), + be16_to_cpu(((__be16 *) sport->gid.raw)[4]), + be16_to_cpu(((__be16 *) sport->gid.raw)[5]), + be16_to_cpu(((__be16 *) sport->gid.raw)[6]), + be16_to_cpu(((__be16 *) sport->gid.raw)[7])); + sport->srpt_tgt.scst_tgt = scst_register_target(&srpt_template, + tgt_name); + if (sport->srpt_tgt.scst_tgt) + scst_tgt_set_tgt_priv(sport->srpt_tgt.scst_tgt, sport); + else + PRINT_ERROR("Registration of target %s failed.", + tgt_name); + } + TRACE_EXIT_RES(0); return 0; @@ -2012,6 +2044,7 @@ static int srpt_compl_thread(void *arg) { struct srpt_rdma_ch *ch; struct srpt_device *sdev; + struct srpt_tgt *srpt_tgt; /* Hibernation / freezing of the SRPT kernel thread is not supported. */ current->flags |= PF_NOFREEZE; @@ -2072,9 +2105,10 @@ static int srpt_compl_thread(void *arg) sdev, ch->rq_size, ch->max_rsp_size, DMA_TO_DEVICE); - spin_lock_irq(&sdev->spinlock); + srpt_tgt = ch->srpt_tgt; + spin_lock_irq(&srpt_tgt->spinlock); list_del(&ch->list); - spin_unlock_irq(&sdev->spinlock); + spin_unlock_irq(&srpt_tgt->spinlock); /* * Note: if a DREQ is received after ch->dreq_received has been read, @@ -2097,7 +2131,7 @@ static int srpt_compl_thread(void *arg) kref_put(&ch->kref, srpt_free_ch); - wake_up(&sdev->ch_releaseQ); + wake_up(&srpt_tgt->ch_releaseQ); return 0; } @@ -2217,13 +2251,13 @@ static bool __srpt_close_ch(struct srpt_rdma_ch *ch) __releases(&ch->srpt_tgt->spinlock) __acquires(&ch->srpt_tgt->spinlock) { - struct srpt_device *sdev = ch->sport->sdev; + struct srpt_tgt *srpt_tgt = ch->srpt_tgt; enum rdma_ch_state prev_state; int ret; bool was_live; #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) - lockdep_assert_held(&sdev->spinlock); + lockdep_assert_held(&srpt_tgt->spinlock); #endif was_live = false; @@ -2242,7 +2276,7 @@ static bool __srpt_close_ch(struct srpt_rdma_ch *ch) if (was_live) { kref_get(&ch->kref); - spin_unlock_irq(&sdev->spinlock); + spin_unlock_irq(&srpt_tgt->spinlock); ret = srpt_ch_qp_err(ch); if (ret < 0) @@ -2250,7 +2284,7 @@ static bool __srpt_close_ch(struct srpt_rdma_ch *ch) " failed: %d", ret); kref_put(&ch->kref, srpt_free_ch); - spin_lock_irq(&sdev->spinlock); + spin_lock_irq(&srpt_tgt->spinlock); } return was_live; @@ -2261,12 +2295,11 @@ static bool __srpt_close_ch(struct srpt_rdma_ch *ch) */ static void srpt_close_ch(struct srpt_rdma_ch *ch) { - struct srpt_device *sdev; + struct srpt_tgt *srpt_tgt = ch->srpt_tgt; - sdev = ch->sport->sdev; - spin_lock_irq(&sdev->spinlock); + spin_lock_irq(&srpt_tgt->spinlock); __srpt_close_ch(ch); - spin_unlock_irq(&sdev->spinlock); + spin_unlock_irq(&srpt_tgt->spinlock); } /** @@ -2297,43 +2330,61 @@ static void srpt_drain_channel(struct ib_cm_id *cm_id) } } -static void __srpt_close_all_ch(struct srpt_device *sdev) +static void __srpt_close_all_ch(struct srpt_tgt *srpt_tgt) { struct srpt_rdma_ch *ch, *next_ch; #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) - lockdep_assert_held(&sdev->spinlock); + lockdep_assert_held(&srpt_tgt->spinlock); #endif - list_for_each_entry_safe(ch, next_ch, &sdev->rch_list, list) { + list_for_each_entry_safe(ch, next_ch, &srpt_tgt->rch_list, list) { PRINT_INFO("Closing channel %s because target %s has been" " disabled", ch->sess_name, - sdev->scst_tgt->tgt_name); + srpt_tgt->scst_tgt->tgt_name); __srpt_close_ch(ch); } } #if !defined(CONFIG_SCST_PROC) +static struct srpt_tgt* srpt_convert_scst_tgt(struct scst_tgt *scst_tgt) +{ + struct srpt_device *sdev; + struct srpt_port *sport; + struct srpt_tgt *srpt_tgt; + + if (one_target_per_port) { + sport = scst_tgt_get_tgt_priv(scst_tgt); + BUG_ON(!sport); + srpt_tgt = &sport->srpt_tgt; + } else { + sdev = scst_tgt_get_tgt_priv(scst_tgt); + BUG_ON(!sdev); + srpt_tgt = &sdev->srpt_tgt; + } + return srpt_tgt; +} + /** * srpt_enable_target - Set the "enabled" status of a target. */ static int srpt_enable_target(struct scst_tgt *scst_tgt, bool enable) { - struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt); + struct srpt_tgt *srpt_tgt = srpt_convert_scst_tgt(scst_tgt); EXTRACHECKS_WARN_ON_ONCE(irqs_disabled()); - if (!sdev) + if (!srpt_tgt) return -ENOENT; TRACE_DBG("%s target %s", enable ? "Enabling" : "Disabling", - sdev->device->name); + scst_tgt->tgt_name); - spin_lock_irq(&sdev->spinlock); - sdev->enabled = enable; + spin_lock_irq(&srpt_tgt->spinlock); + srpt_tgt->enabled = enable; if (!enable) - __srpt_close_all_ch(sdev); - spin_unlock_irq(&sdev->spinlock); + __srpt_close_all_ch(srpt_tgt); + spin_unlock_irq(&srpt_tgt->spinlock); return 0; } @@ -2343,18 +2394,9 @@ static int srpt_enable_target(struct scst_tgt *scst_tgt, bool enable) */ static bool srpt_is_target_enabled(struct scst_tgt *scst_tgt) { - struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt); - bool res; + struct srpt_tgt *srpt_tgt = srpt_convert_scst_tgt(scst_tgt); - EXTRACHECKS_WARN_ON_ONCE(irqs_disabled()); - - if (!sdev) - return false; - - spin_lock_irq(&sdev->spinlock); - res = sdev->enabled; - spin_unlock_irq(&sdev->spinlock); - return res; + return srpt_tgt && srpt_tgt->enabled; } #endif @@ -2368,7 +2410,10 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, struct ib_cm_req_event_param *param, void *private_data) { - struct srpt_device *sdev = cm_id->context; + struct srpt_device *const sdev = cm_id->context; + struct srpt_port *const sport = &sdev->port[param->port - 1]; + struct srpt_tgt *const srpt_tgt = one_target_per_port ? + &sport->srpt_tgt : &sdev->srpt_tgt; struct srp_login_req *req; struct srp_login_rsp *rsp; struct srp_login_rej *rej; @@ -2443,12 +2488,11 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, goto reject; } - if (!sdev->enabled) { + if (!srpt_tgt->enabled) { rej->reason = cpu_to_be32( SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); - PRINT_ERROR("rejected SRP_LOGIN_REQ because the target %s (%s)" - " is not enabled", - sdev->scst_tgt->tgt_name, sdev->device->name); + PRINT_ERROR("rejected SRP_LOGIN_REQ because target %s is not" + " enabled", srpt_tgt->scst_tgt->tgt_name); goto reject; } @@ -2473,7 +2517,8 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, kref_init(&ch->kref); memcpy(ch->i_port_id, req->initiator_port_id, 16); memcpy(ch->t_port_id, req->target_port_id, 16); - ch->sport = &sdev->port[param->port - 1]; + ch->sport = sport; + ch->srpt_tgt = srpt_tgt; ch->cm_id = cm_id; cm_id->context = ch; /* @@ -2509,7 +2554,20 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, goto free_ring; } - if (use_port_guid_in_session_name) { + if (one_target_per_port) { + __be16 *const raw_gid = (__be16 *)param->primary_path->dgid.raw; + + snprintf(ch->sess_name, sizeof(ch->sess_name), + "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x", + be16_to_cpu(raw_gid[0]), + be16_to_cpu(raw_gid[1]), + be16_to_cpu(raw_gid[2]), + be16_to_cpu(raw_gid[3]), + be16_to_cpu(raw_gid[4]), + be16_to_cpu(raw_gid[5]), + be16_to_cpu(raw_gid[6]), + be16_to_cpu(raw_gid[7])); + } else if (use_port_guid_in_session_name) { /* * If the kernel module parameter use_port_guid_in_session_name * has been specified, use a combination of the target port @@ -2535,9 +2593,9 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, TRACE_DBG("registering session %s", ch->sess_name); - BUG_ON(!sdev->scst_tgt); + BUG_ON(!srpt_tgt->scst_tgt); ret = -ENOMEM; - ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name, + ch->scst_sess = scst_register_session(srpt_tgt->scst_tgt, 0, ch->sess_name, ch, NULL, NULL); if (!ch->scst_sess) { rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); @@ -2545,8 +2603,8 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, goto destroy_ib; } - thread = kthread_run(srpt_compl_thread, ch, "srpt_%s", - ch->sport->sdev->device->name); + thread = kthread_run(srpt_compl_thread, ch, "srpt_%s-%d", + ch->sport->sdev->device->name, ch->sport->port); if (IS_ERR(thread)) { rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); ret = PTR_ERR(thread); @@ -2554,14 +2612,14 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, goto unreg_ch; } - spin_lock_irq(&sdev->spinlock); + spin_lock_irq(&srpt_tgt->spinlock); if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) { struct srpt_rdma_ch *ch2; rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN; restart: - list_for_each_entry(ch2, &sdev->rch_list, list) { + list_for_each_entry(ch2, &srpt_tgt->rch_list, list) { if (!memcmp(ch2->i_port_id, req->initiator_port_id, 16) && param->port == ch2->sport->port && param->listen_id == ch2->sport->sdev->cm_id) { @@ -2582,20 +2640,20 @@ restart: rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED; } - list_add_tail(&ch->list, &sdev->rch_list); + list_add_tail(&ch->list, &srpt_tgt->rch_list); ch->thread = thread; - if (!sdev->enabled) { + if (!srpt_tgt->enabled) { rej->reason = cpu_to_be32( SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); PRINT_ERROR("rejected SRP_LOGIN_REQ because the target %s (%s)" " is not enabled", - sdev->scst_tgt->tgt_name, sdev->device->name); - spin_unlock_irq(&sdev->spinlock); + srpt_tgt->scst_tgt->tgt_name, sdev->device->name); + spin_unlock_irq(&srpt_tgt->spinlock); goto reject; } - spin_unlock_irq(&sdev->spinlock); + spin_unlock_irq(&srpt_tgt->spinlock); ret = srpt_ch_qp_rtr(ch, ch->qp); if (ret) { @@ -2631,12 +2689,12 @@ restart: rep_param->responder_resources = 4; rep_param->initiator_depth = 4; - spin_lock_irq(&sdev->spinlock); + spin_lock_irq(&srpt_tgt->spinlock); if (ch->state == CH_CONNECTING) ret = ib_send_cm_rep(cm_id, rep_param); else ret = -ECONNABORTED; - spin_unlock_irq(&sdev->spinlock); + spin_unlock_irq(&srpt_tgt->spinlock); switch (ret) { case 0: @@ -3440,46 +3498,46 @@ static int srpt_detect(struct scst_tgt_template *tp) return device_count; } -static int srpt_ch_list_empty(struct srpt_device *sdev) +static int srpt_ch_list_empty(struct srpt_tgt *srpt_tgt) { int res; - spin_lock_irq(&sdev->spinlock); - res = list_empty(&sdev->rch_list); - spin_unlock_irq(&sdev->spinlock); + spin_lock_irq(&srpt_tgt->spinlock); + res = list_empty(&srpt_tgt->rch_list); + spin_unlock_irq(&srpt_tgt->spinlock); return res; } /** - * srpt_release_sdev() - Free channel resources associated with a target. + * srpt_release_sport() - Free channel resources associated with a target. */ -static int srpt_release_sdev(struct srpt_device *sdev) +static int srpt_release_sport(struct srpt_tgt *srpt_tgt) { struct srpt_rdma_ch *ch; TRACE_ENTRY(); WARN_ON_ONCE(irqs_disabled()); - BUG_ON(!sdev); + BUG_ON(!srpt_tgt); /* Disallow new logins and close all active sessions. */ - spin_lock_irq(&sdev->spinlock); - sdev->enabled = false; - __srpt_close_all_ch(sdev); - spin_unlock_irq(&sdev->spinlock); + spin_lock_irq(&srpt_tgt->spinlock); + srpt_tgt->enabled = false; + __srpt_close_all_ch(srpt_tgt); + spin_unlock_irq(&srpt_tgt->spinlock); - while (wait_event_timeout(sdev->ch_releaseQ, - srpt_ch_list_empty(sdev), 5 * HZ) <= 0) { + while (wait_event_timeout(srpt_tgt->ch_releaseQ, + srpt_ch_list_empty(srpt_tgt), 5 * HZ) <= 0) { PRINT_INFO("%s: waiting for session unregistration ...", - sdev->device->name); - spin_lock_irq(&sdev->spinlock); - list_for_each_entry(ch, &sdev->rch_list, list) { + srpt_tgt->scst_tgt->tgt_name); + spin_lock_irq(&srpt_tgt->spinlock); + list_for_each_entry(ch, &srpt_tgt->rch_list, list) { PRINT_INFO("%s: state %s; %d commands in progress", ch->sess_name, get_ch_state_name(ch->state), atomic_read(&ch->scst_sess->sess_cmd_count)); } - spin_unlock_irq(&sdev->spinlock); + spin_unlock_irq(&srpt_tgt->spinlock); } TRACE_EXIT(); @@ -3493,7 +3551,7 @@ static int srpt_release_sdev(struct srpt_device *sdev) */ static int srpt_release(struct scst_tgt *scst_tgt) { - struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt); + struct srpt_tgt *srpt_tgt = srpt_convert_scst_tgt(scst_tgt); TRACE_ENTRY(); @@ -3501,15 +3559,15 @@ static int srpt_release(struct scst_tgt *scst_tgt) BUG_ON(!scst_tgt); #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18) - WARN_ON(!sdev); - if (!sdev) + WARN_ON(!srpt_tgt); + if (!srpt_tgt) return -ENODEV; #else - if (WARN_ON(!sdev)) + if (WARN_ON(!srpt_tgt)) return -ENODEV; #endif - srpt_release_sdev(sdev); + srpt_release_sport(srpt_tgt); scst_tgt_set_tgt_priv(scst_tgt, NULL); @@ -3532,33 +3590,52 @@ static uint16_t srpt_get_scsi_transport_version(struct scst_tgt *scst_tgt) static ssize_t show_login_info(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - struct scst_tgt *scst_tgt; - struct srpt_device *sdev; + struct scst_tgt *scst_tgt = container_of(kobj, struct scst_tgt, + tgt_kobj); + struct srpt_tgt *srpt_tgt = srpt_convert_scst_tgt(scst_tgt); struct srpt_port *sport; - int i; - int len; + int i, len; - scst_tgt = container_of(kobj, struct scst_tgt, tgt_kobj); - sdev = scst_tgt_get_tgt_priv(scst_tgt); - len = 0; - for (i = 0; i < sdev->device->phys_port_cnt; i++) { - sport = &sdev->port[i]; + if (one_target_per_port) { + sport = container_of(srpt_tgt, struct srpt_port, srpt_tgt); + len = sprintf(buf, + "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff," + "dgid=%04x%04x%04x%04x%04x%04x%04x%04x," + "service_id=%016llx\n", + srpt_service_guid, srpt_service_guid, + be16_to_cpu(((__be16 *) sport->gid.raw)[0]), + be16_to_cpu(((__be16 *) sport->gid.raw)[1]), + be16_to_cpu(((__be16 *) sport->gid.raw)[2]), + be16_to_cpu(((__be16 *) sport->gid.raw)[3]), + be16_to_cpu(((__be16 *) sport->gid.raw)[4]), + be16_to_cpu(((__be16 *) sport->gid.raw)[5]), + be16_to_cpu(((__be16 *) sport->gid.raw)[6]), + be16_to_cpu(((__be16 *) sport->gid.raw)[7]), + srpt_service_guid); + } else { + struct srpt_device *sdev; - len += sprintf(buf + len, - "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff," - "dgid=%04x%04x%04x%04x%04x%04x%04x%04x," - "service_id=%016llx\n", - srpt_service_guid, - srpt_service_guid, - be16_to_cpu(((__be16 *) sport->gid.raw)[0]), - be16_to_cpu(((__be16 *) sport->gid.raw)[1]), - be16_to_cpu(((__be16 *) sport->gid.raw)[2]), - be16_to_cpu(((__be16 *) sport->gid.raw)[3]), - be16_to_cpu(((__be16 *) sport->gid.raw)[4]), - be16_to_cpu(((__be16 *) sport->gid.raw)[5]), - be16_to_cpu(((__be16 *) sport->gid.raw)[6]), - be16_to_cpu(((__be16 *) sport->gid.raw)[7]), - srpt_service_guid); + sdev = container_of(srpt_tgt, struct srpt_device, srpt_tgt); + len = 0; + for (i = 0; i < sdev->device->phys_port_cnt; i++) { + sport = &sdev->port[i]; + + len += sprintf(buf + len, + "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff," + "dgid=%04x%04x%04x%04x%04x%04x%04x%04x," + "service_id=%016llx\n", + srpt_service_guid, + srpt_service_guid, + be16_to_cpu(((__be16 *) sport->gid.raw)[0]), + be16_to_cpu(((__be16 *) sport->gid.raw)[1]), + be16_to_cpu(((__be16 *) sport->gid.raw)[2]), + be16_to_cpu(((__be16 *) sport->gid.raw)[3]), + be16_to_cpu(((__be16 *) sport->gid.raw)[4]), + be16_to_cpu(((__be16 *) sport->gid.raw)[5]), + be16_to_cpu(((__be16 *) sport->gid.raw)[6]), + be16_to_cpu(((__be16 *) sport->gid.raw)[7]), + srpt_service_guid); + } } return len; @@ -3682,6 +3759,7 @@ static void srpt_add_one(struct ib_device *device) { struct srpt_device *sdev; struct srpt_port *sport; + struct srpt_tgt *srpt_tgt; struct ib_srq_init_attr srq_attr; char tgt_name[24]; int i, ret; @@ -3695,27 +3773,35 @@ static void srpt_add_one(struct ib_device *device) goto err; sdev->device = device; - INIT_LIST_HEAD(&sdev->rch_list); - init_waitqueue_head(&sdev->ch_releaseQ); - spin_lock_init(&sdev->spinlock); - if (use_node_guid_in_target_name) { - snprintf(tgt_name, sizeof(tgt_name), "%04x:%04x:%04x:%04x", - be16_to_cpu(((__be16 *)&device->node_guid)[0]), - be16_to_cpu(((__be16 *)&device->node_guid)[1]), - be16_to_cpu(((__be16 *)&device->node_guid)[2]), - be16_to_cpu(((__be16 *)&device->node_guid)[3])); - sdev->scst_tgt = scst_register_target(&srpt_template, tgt_name); - } else - sdev->scst_tgt = scst_register_target(&srpt_template, NULL); - if (!sdev->scst_tgt) { - PRINT_ERROR("SCST registration failed for %s.", - sdev->device->name); - goto free_dev; + if (!one_target_per_port) { + srpt_tgt = &sdev->srpt_tgt; + INIT_LIST_HEAD(&srpt_tgt->rch_list); + init_waitqueue_head(&srpt_tgt->ch_releaseQ); + spin_lock_init(&srpt_tgt->spinlock); + + if (use_node_guid_in_target_name) { + snprintf(tgt_name, sizeof(tgt_name), + "%04x:%04x:%04x:%04x", + be16_to_cpu(((__be16 *)&device->node_guid)[0]), + be16_to_cpu(((__be16 *)&device->node_guid)[1]), + be16_to_cpu(((__be16 *)&device->node_guid)[2]), + be16_to_cpu(((__be16 *)&device->node_guid)[3])); + srpt_tgt->scst_tgt = + scst_register_target(&srpt_template, tgt_name); + } else { + srpt_tgt->scst_tgt = + scst_register_target(&srpt_template, NULL); + } + if (!srpt_tgt->scst_tgt) { + PRINT_ERROR("SCST registration failed for %s.", + sdev->device->name); + goto free_dev; + } + + scst_tgt_set_tgt_priv(srpt_tgt->scst_tgt, sdev); } - scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev); - ret = ib_query_device(device, &sdev->dev_attr); if (ret) { PRINT_ERROR("ib_query_device() failed: %d", ret); @@ -3812,6 +3898,11 @@ static void srpt_add_one(struct ib_device *device) sport = &sdev->port[i - 1]; sport->sdev = sdev; sport->port = i; + if (one_target_per_port) { + INIT_LIST_HEAD(&sport->srpt_tgt.rch_list); + init_waitqueue_head(&sport->srpt_tgt.ch_releaseQ); + spin_lock_init(&sport->srpt_tgt.spinlock); + } #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && !defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19) /* * A vanilla 2.6.19 or older kernel without backported OFED @@ -3850,7 +3941,8 @@ err_mr: err_pd: ib_dealloc_pd(sdev->pd); unregister_tgt: - scst_unregister_target(sdev->scst_tgt); + if (!one_target_per_port) + scst_unregister_target(sdev->srpt_tgt.scst_tgt); free_dev: kfree(sdev); err: @@ -3895,12 +3987,23 @@ static void srpt_remove_one(struct ib_device *device) ib_destroy_cm_id(sdev->cm_id); /* - * Unregistering an SCST target must happen after destroying sdev->cm_id + * SCST target unregistration must happen after destroying sdev->cm_id * such that no new SRP_LOGIN_REQ information units can arrive while - * destroying the SCST target. + * unregistering the SCST target. */ - scst_unregister_target(sdev->scst_tgt); - sdev->scst_tgt = NULL; + if (one_target_per_port) { + for (i = 0; i < sdev->device->phys_port_cnt; i++) { + struct srpt_tgt *tgt = &sdev->port[i].srpt_tgt; + + if (tgt->scst_tgt) { + scst_unregister_target(tgt->scst_tgt); + tgt->scst_tgt = NULL; + } + } + } else { + scst_unregister_target(sdev->srpt_tgt.scst_tgt); + sdev->srpt_tgt.scst_tgt = NULL; + } ib_destroy_srq(sdev->srq); ib_dereg_mr(sdev->mr); @@ -4016,13 +4119,14 @@ static int __init srpt_init_module(void) goto out; } - if (!use_node_guid_in_target_name) - PRINT_WARNING("%s", "Usage of HCA numbers as SCST target names " - "is deprecated and will be removed in one of the next " - "versions. It is strongly recommended to set " - "use_node_guid_in_target_name parameter in 1 and " - "update your SCST config file accordingly to use HCAs " - "GUIDs."); + if (!one_target_per_port) + PRINT_WARNING("%s%s", !use_node_guid_in_target_name ? + "Using one target per HCA " : + "Using autogenerated target names ", + "is deprecated and will be removed in one of the " + "next versions. It is strongly recommended to " + "set the one_target_per_port parameter to true " + "and to update your SCST config file."); ret = scst_register_target_template(&srpt_template); if (ret < 0) { diff --git a/srpt/src/ib_srpt.h b/srpt/src/ib_srpt.h index 28a6221a8..1f1454c76 100644 --- a/srpt/src/ib_srpt.h +++ b/srpt/src/ib_srpt.h @@ -56,6 +56,8 @@ */ #define SRP_SERVICE_NAME_PREFIX "SRP.T10:" +struct srpt_tgt; + enum { /* * SRP IOControllerProfile attributes for SRP target ports that have @@ -327,6 +329,7 @@ struct srpt_rdma_ch { int max_rsp_size; int sq_wr_avail; struct srpt_port *sport; + struct srpt_tgt *srpt_tgt; u8 i_port_id[16]; u8 t_port_id[16]; int max_ti_iu_len; @@ -345,7 +348,23 @@ struct srpt_rdma_ch { bool last_wqe_received; struct scst_session *scst_sess; - u8 sess_name[36]; + u8 sess_name[40]; +}; + +/** + * struct srpt_tgt + * @ch_releaseQ: Enables waiting for removal from rch_list. + * @spinlock: Protects rch_list. + * @rch_list: Per-device channel list -- see also srpt_rdma_ch.list. + * @scst_tgt: SCST target information associated with this HCA. + * @enabled: Whether or not this SCST target is enabled. + */ +struct srpt_tgt { + wait_queue_head_t ch_releaseQ; + spinlock_t spinlock; + struct list_head rch_list; + struct scst_tgt *scst_tgt; + bool enabled; }; /** @@ -359,13 +378,14 @@ struct srpt_rdma_ch { * @work: work structure for refreshing the aforementioned cached values. */ struct srpt_port { - struct srpt_device *sdev; - struct ib_mad_agent *mad_agent; - u8 port; - u16 sm_lid; - u16 lid; - union ib_gid gid; - struct work_struct work; + struct srpt_device *sdev; + struct ib_mad_agent *mad_agent; + u8 port; + u16 sm_lid; + u16 lid; + union ib_gid gid; + struct work_struct work; + struct srpt_tgt srpt_tgt; }; /** @@ -379,14 +399,8 @@ struct srpt_port { * ib_client.add() callback. * @srq_size: SRQ size. * @ioctx_ring: Per-HCA SRQ. - * @rch_list: Per-device channel list -- see also srpt_rdma_ch.list. - * @ch_releaseQ: Enables waiting for removal from rch_list. - * @spinlock: Protects rch_list. * @port: Information about the ports owned by this HCA. * @event_handler: Per-HCA asynchronous IB event handler. - * @dev: Per-port srpt- device instance. - * @scst_tgt: SCST target information associated with this HCA. - * @enabled: Whether or not this SCST target is enabled. */ struct srpt_device { struct ib_device *device; @@ -397,13 +411,9 @@ struct srpt_device { struct ib_device_attr dev_attr; int srq_size; struct srpt_recv_ioctx **ioctx_ring; - struct list_head rch_list; - wait_queue_head_t ch_releaseQ; - spinlock_t spinlock; struct srpt_port port[2]; struct ib_event_handler event_handler; - struct scst_tgt *scst_tgt; - bool enabled; + struct srpt_tgt srpt_tgt; }; #endif /* IB_SRPT_H */