diff --git a/cmd/vgwrdma/main.go b/cmd/vgwrdma/main.go index 753de36d..01e3e592 100644 --- a/cmd/vgwrdma/main.go +++ b/cmd/vgwrdma/main.go @@ -115,6 +115,7 @@ var ( socketPerm string rdmaIP string rcGidHint string + rcDevice string rdmaRCEnable bool rdmaPort uint poolBufSize int @@ -905,6 +906,12 @@ func initFlags() []cli.Flag { EnvVars: []string{"VGW_RDMA_IP"}, Destination: &rdmaIP, }, + &cli.StringFlag{ + Name: "rc-device", + Usage: "verbs device name (e.g. mlx5_0) for the hipobj-rc-v2 RC data plane; takes precedence over --rc-gid-hint", + EnvVars: []string{"VGW_RC_DEVICE"}, + Destination: &rcDevice, + }, &cli.StringFlag{ Name: "rc-gid-hint", Usage: "dotted GID prefix selecting the verbs device for the hipobj-rc-v2 RC data plane (default: first device)", @@ -1264,6 +1271,7 @@ func runGateway(ctx context.Context, be backend.Backend) error { rcSvc, err := rcserver.Init(rcserver.DeviceOpts{ GidHint: rcGidHint, + DevName: rcDevice, Port: 1, MaxSessions: uint32(rcMaxSessions), MaxUserSessions: uint32(rcMaxUserSessions), diff --git a/cuwrapper/rc/rc_server_abi.cpp b/cuwrapper/rc/rc_server_abi.cpp index e9787db7..bbc7a42d 100644 --- a/cuwrapper/rc/rc_server_abi.cpp +++ b/cuwrapper/rc/rc_server_abi.cpp @@ -346,11 +346,17 @@ int rc_server_init(const rc_device_opts *opts, rc_server **out) { return RC_E_INTERNAL; } struct ibv_device *chosen = devs[0]; - /* GID hint: pick the first device/port whose GID starts with it. - * Query with srv->opts.port, which the normalization above has - * already made 1-based. */ + /* Device name, when given, is the primary filter: on a host with + * several HCAs (an IB adapter next to a RoCE NIC, say) neither + * "first device" nor the 4-byte GID prefix can single out the + * one on the right fabric, since every link-local GID starts + * fe:80:0:0. Then the GID hint: pick the first device/port whose + * GID starts with it. Query with srv->opts.port, which the + * normalization above has already made 1-based. */ + const bool byName = opts->dev_name && *opts->dev_name; struct ibv_context *ctx = nullptr; for (int i = 0; i < n && !ctx; i++) { + if (byName && strcmp(devs[i]->name, opts->dev_name) != 0) continue; struct ibv_context *c = hipObj::ibv.open_device(devs[i]); if (!c) continue; if (opts->gid_hint) { @@ -378,8 +384,14 @@ int rc_server_init(const rc_device_opts *opts, rc_server **out) { } if (!ctx) { hipObj::ibv.free_device_list(devs); - fprintf(stderr, "rc: no verbs device matches gid_hint %.32s\n", - opts->gid_hint ? opts->gid_hint : ""); + if (byName) { + fprintf(stderr, "rc: no verbs device named %.64s%s\n", + opts->dev_name, + opts->gid_hint ? " with a GID matching gid_hint" : ""); + } else { + fprintf(stderr, "rc: no verbs device matches gid_hint %.32s\n", + opts->gid_hint ? opts->gid_hint : ""); + } return RC_E_INTERNAL; } struct ibv_pd *pd = hipObj::ibv.alloc_pd(ctx); diff --git a/cuwrapper/rc/rc_server_abi.h b/cuwrapper/rc/rc_server_abi.h index 9193524a..1ba389ae 100644 --- a/cuwrapper/rc/rc_server_abi.h +++ b/cuwrapper/rc/rc_server_abi.h @@ -67,10 +67,12 @@ typedef void (*rc_terminal_fn)(void *ctx, const char *id, int outcome, void rc_server_set_terminal_notify(rc_server *srv, rc_terminal_fn fn, void *ctx); -/* Device selection: matching GID prefix when gid_hint is set, - * otherwise the first verbs device. */ +/* Device selection: the verbs device named by dev_name when set, + * else the first device whose GID starts with gid_hint when that is + * set, otherwise the first verbs device that opens. */ typedef struct { const char *gid_hint; /* nullable; dotted GID prefix */ + const char *dev_name; /* nullable; verbs device name, e.g. mlx5_0 */ uint8_t port; int gid_index; uint32_t max_sessions; diff --git a/rdma/rcserver/rcserver_linux.go b/rdma/rcserver/rcserver_linux.go index 72d16c8e..7614b21c 100644 --- a/rdma/rcserver/rcserver_linux.go +++ b/rdma/rcserver/rcserver_linux.go @@ -71,6 +71,11 @@ type DeviceOpts struct { // GidHint selects the device whose GID starts with this dotted // prefix; empty picks the first verbs device. GidHint string + // DevName selects the verbs device by name (e.g. mlx5_0). It takes + // precedence over GidHint and the first-device default, which + // cannot tell HCAs apart on a host where every port's GID 0 is + // link-local. + DevName string Port uint8 GidIdx int // Global limits. @@ -256,6 +261,12 @@ func Init(opts DeviceOpts) (*RCSvc, error) { defer C.free(unsafe.Pointer(gidHint)) } copts.gid_hint = gidHint + var devName *C.char + if opts.DevName != "" { + devName = C.CString(opts.DevName) + defer C.free(unsafe.Pointer(devName)) + } + copts.dev_name = devName var srv *C.rc_server if rc := C.rc_server_init(&copts, &srv); rc != C.RC_OK { diff --git a/rdma/rcserver/rcserver_stub.go b/rdma/rcserver/rcserver_stub.go index ef546320..d570716e 100644 --- a/rdma/rcserver/rcserver_stub.go +++ b/rdma/rcserver/rcserver_stub.go @@ -25,6 +25,7 @@ var errNotSupported = errors.New("rcserver: not supported on this platform") // DeviceOpts configures device selection and resource limits. type DeviceOpts struct { GidHint string + DevName string Port uint8 GidIdx int MaxSessions uint32