Merge pull request #2442 from hgichon/fix/rc-device-select

rdma: add --rc-device to pick the RC data plane's verbs device by name

cmd/vgwrdma/main.go

@@ -115,6 +115,7 @@ var (
 	socketPerm                             string
 	rdmaIP                                 string
 	rcGidHint                              string
+	rcDevice                               string
 	rdmaRCEnable                           bool
 	rdmaPort                               uint
 	poolBufSize                            int
@@ -905,6 +906,12 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_RDMA_IP"},
 			Destination: &rdmaIP,
 		},
+		&cli.StringFlag{
+			Name:        "rc-device",
+			Usage:       "verbs device name (e.g. mlx5_0) for the hipobj-rc-v2 RC data plane; takes precedence over --rc-gid-hint",
+			EnvVars:     []string{"VGW_RC_DEVICE"},
+			Destination: &rcDevice,
+		},
 		&cli.StringFlag{
 			Name:        "rc-gid-hint",
 			Usage:       "dotted GID prefix selecting the verbs device for the hipobj-rc-v2 RC data plane (default: first device)",
@@ -1264,6 +1271,7 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 
 		rcSvc, err := rcserver.Init(rcserver.DeviceOpts{
 			GidHint:             rcGidHint,
+			DevName:             rcDevice,
 			Port:                1,
 			MaxSessions:         uint32(rcMaxSessions),
 			MaxUserSessions:     uint32(rcMaxUserSessions),

cuwrapper/rc/rc_server_abi.cpp

@@ -346,11 +346,17 @@ int rc_server_init(const rc_device_opts *opts, rc_server **out) {
     return RC_E_INTERNAL;
   }
   struct ibv_device *chosen = devs[0];
-  /* GID hint: pick the first device/port whose GID starts with it.
-   * Query with srv->opts.port, which the normalization above has
-   * already made 1-based. */
+  /* Device name, when given, is the primary filter: on a host with
+   * several HCAs (an IB adapter next to a RoCE NIC, say) neither
+   * "first device" nor the 4-byte GID prefix can single out the
+   * one on the right fabric, since every link-local GID starts
+   * fe:80:0:0. Then the GID hint: pick the first device/port whose
+   * GID starts with it. Query with srv->opts.port, which the
+   * normalization above has already made 1-based. */
+  const bool byName = opts->dev_name && *opts->dev_name;
   struct ibv_context *ctx = nullptr;
   for (int i = 0; i < n && !ctx; i++) {
+    if (byName && strcmp(devs[i]->name, opts->dev_name) != 0) continue;
     struct ibv_context *c = hipObj::ibv.open_device(devs[i]);
     if (!c) continue;
     if (opts->gid_hint) {
@@ -378,8 +384,14 @@ int rc_server_init(const rc_device_opts *opts, rc_server **out) {
   }
   if (!ctx) {
     hipObj::ibv.free_device_list(devs);
-    fprintf(stderr, "rc: no verbs device matches gid_hint %.32s\n",
-            opts->gid_hint ? opts->gid_hint : "");
+    if (byName) {
+      fprintf(stderr, "rc: no verbs device named %.64s%s\n",
+              opts->dev_name,
+              opts->gid_hint ? " with a GID matching gid_hint" : "");
+    } else {
+      fprintf(stderr, "rc: no verbs device matches gid_hint %.32s\n",
+              opts->gid_hint ? opts->gid_hint : "");
+    }
     return RC_E_INTERNAL;
   }
   struct ibv_pd *pd = hipObj::ibv.alloc_pd(ctx);

cuwrapper/rc/rc_server_abi.h

@@ -67,10 +67,12 @@ typedef void (*rc_terminal_fn)(void *ctx, const char *id, int outcome,
 void rc_server_set_terminal_notify(rc_server *srv, rc_terminal_fn fn,
                                    void *ctx);
 
-/* Device selection: matching GID prefix when gid_hint is set,
- * otherwise the first verbs device. */
+/* Device selection: the verbs device named by dev_name when set,
+ * else the first device whose GID starts with gid_hint when that is
+ * set, otherwise the first verbs device that opens. */
 typedef struct {
   const char *gid_hint; /* nullable; dotted GID prefix */
+  const char *dev_name; /* nullable; verbs device name, e.g. mlx5_0 */
   uint8_t port;
   int gid_index;
   uint32_t max_sessions;

rdma/rcserver/rcserver_linux.go

@@ -71,6 +71,11 @@ type DeviceOpts struct {
 	// GidHint selects the device whose GID starts with this dotted
 	// prefix; empty picks the first verbs device.
 	GidHint string
+	// DevName selects the verbs device by name (e.g. mlx5_0). It takes
+	// precedence over GidHint and the first-device default, which
+	// cannot tell HCAs apart on a host where every port's GID 0 is
+	// link-local.
+	DevName string
 	Port    uint8
 	GidIdx  int
 	// Global limits.
@@ -256,6 +261,12 @@ func Init(opts DeviceOpts) (*RCSvc, error) {
 		defer C.free(unsafe.Pointer(gidHint))
 	}
 	copts.gid_hint = gidHint
+	var devName *C.char
+	if opts.DevName != "" {
+		devName = C.CString(opts.DevName)
+		defer C.free(unsafe.Pointer(devName))
+	}
+	copts.dev_name = devName
 
 	var srv *C.rc_server
 	if rc := C.rc_server_init(&copts, &srv); rc != C.RC_OK {

rdma/rcserver/rcserver_stub.go

@@ -25,6 +25,7 @@ var errNotSupported = errors.New("rcserver: not supported on this platform")
 // DeviceOpts configures device selection and resource limits.
 type DeviceOpts struct {
 	GidHint             string
+	DevName             string
 	Port                uint8
 	GidIdx              int
 	MaxSessions         uint32