diff --git a/cuobjclient/session_host_linux.go b/cuobjclient/session_host_linux.go index 57ce5a1a..b1505be7 100644 --- a/cuobjclient/session_host_linux.go +++ b/cuobjclient/session_host_linux.go @@ -24,8 +24,9 @@ // // VGWRDMA_RDMA_DEV RDMA device name (default: first device, e.g. mlx5_0) // VGWRDMA_RDMA_PORT HCA port number (default: 1) -// VGWRDMA_GID_INDEX RoCE GID index from `ibv_devinfo -v` -// (default: auto-select first non-link-local GID) +// VGWRDMA_GID_INDEX GID index from `ibv_devinfo -v` +// (default: RoCE auto-selects the first non-link-local +// GID, preferring RoCEv2; InfiniBand uses index 0) // VGWRDMA_DC_KEY Dynamic Connection key, decimal or 0x-hex // (default: matches server DCKey 0xffeeddcc) package cuobjclient diff --git a/cuwrapper/rdma_host_client_wrapper.cpp b/cuwrapper/rdma_host_client_wrapper.cpp index 5e48e708..e639e848 100644 --- a/cuwrapper/rdma_host_client_wrapper.cpp +++ b/cuwrapper/rdma_host_client_wrapper.cpp @@ -112,6 +112,22 @@ static bool select_gid(rdma_host_client_t *c) { return false; } + // Native InfiniBand ports carry no RoCE GID pairs: entry 0 is the + // port GID (subnet prefix + port GUID) and it is deliberately + // link-local (fe80::/10). The RoCE preference order below skips + // link-local entries, so on IB it never finds a candidate and the + // caller has to hard-code VGWRDMA_GID_INDEX=0. Select entry 0 here. + if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { + union ibv_gid gid = {}; + if (ibv_query_gid(c->ctx, c->port_num, 0, &gid) || gid_is_zero(gid)) { + set_err(c, "InfiniBand port has no GID at index 0"); + return false; + } + c->gid_index = 0; + c->gid = gid; + return true; + } + int fallback_gid_index = -1; union ibv_gid fallback_gid = {}; int v1_fallback_gid_index = -1; @@ -179,8 +195,8 @@ static bool select_gid(rdma_host_client_t *c) { } set_err(c, - "no usable GID found on selected RDMA port; " - "set VGWRDMA_GID_INDEX to a valid RoCE GID index from `ibv_devinfo -v`"); + "no usable GID found on selected RoCE port; " + "set VGWRDMA_GID_INDEX to a valid GID index from `ibv_devinfo -v`"); return false; }