From fd3864d135b7fcfe246c6d825204a636bf1dc161 Mon Sep 17 00:00:00 2001 From: Kyeongpyo Kim Date: Sat, 26 Sep 2026 12:56:29 +0900 Subject: [PATCH] cuobjclient: select GID index 0 on InfiniBand ports in the host client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit select_gid encodes a RoCE preference order: skip zero and link-local entries, prefer a RoCEv2 non-IPv4-mapped GID, fall back to IPv4-mapped v2 and then v1. That order has nothing to pick on a native InfiniBand port. The only populated entry is index 0, the port GID formed from the subnet prefix and the port GUID, and it is link-local by construction (fe80::/10), so the loop discards it and the session fails with "no usable GID found ... set VGWRDMA_GID_INDEX", even though index 0 is the one and only correct choice. Every IB user has to hard-code VGWRDMA_GID_INDEX=0. Check port_attr.link_layer, which ibv_query_port already returns here, and take index 0 on IBV_LINK_LAYER_INFINIBAND before the RoCE scan. RoCE ports are unaffected. Reword the failure message so it no longer tells IB users to look for a "RoCE GID index", and update the env var documentation in cuobjclient. Verified on a ConnectX-6 native IB port (GID table: index 0 = fe80::…, 1-3 zero): the host client now auto-selects index 0 and PUT/GET complete against a hipobj/cuObject gateway without the env var; unpatched it fails at session init with the message above. --- cuobjclient/session_host_linux.go | 5 +++-- cuwrapper/rdma_host_client_wrapper.cpp | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cuobjclient/session_host_linux.go b/cuobjclient/session_host_linux.go index 57ce5a1a..b1505be7 100644 --- a/cuobjclient/session_host_linux.go +++ b/cuobjclient/session_host_linux.go @@ -24,8 +24,9 @@ // // VGWRDMA_RDMA_DEV RDMA device name (default: first device, e.g. mlx5_0) // VGWRDMA_RDMA_PORT HCA port number (default: 1) -// VGWRDMA_GID_INDEX RoCE GID index from `ibv_devinfo -v` -// (default: auto-select first non-link-local GID) +// VGWRDMA_GID_INDEX GID index from `ibv_devinfo -v` +// (default: RoCE auto-selects the first non-link-local +// GID, preferring RoCEv2; InfiniBand uses index 0) // VGWRDMA_DC_KEY Dynamic Connection key, decimal or 0x-hex // (default: matches server DCKey 0xffeeddcc) package cuobjclient diff --git a/cuwrapper/rdma_host_client_wrapper.cpp b/cuwrapper/rdma_host_client_wrapper.cpp index 5e48e708..e639e848 100644 --- a/cuwrapper/rdma_host_client_wrapper.cpp +++ b/cuwrapper/rdma_host_client_wrapper.cpp @@ -112,6 +112,22 @@ static bool select_gid(rdma_host_client_t *c) { return false; } + // Native InfiniBand ports carry no RoCE GID pairs: entry 0 is the + // port GID (subnet prefix + port GUID) and it is deliberately + // link-local (fe80::/10). The RoCE preference order below skips + // link-local entries, so on IB it never finds a candidate and the + // caller has to hard-code VGWRDMA_GID_INDEX=0. Select entry 0 here. + if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { + union ibv_gid gid = {}; + if (ibv_query_gid(c->ctx, c->port_num, 0, &gid) || gid_is_zero(gid)) { + set_err(c, "InfiniBand port has no GID at index 0"); + return false; + } + c->gid_index = 0; + c->gid = gid; + return true; + } + int fallback_gid_index = -1; union ibv_gid fallback_gid = {}; int v1_fallback_gid_index = -1; @@ -179,8 +195,8 @@ static bool select_gid(rdma_host_client_t *c) { } set_err(c, - "no usable GID found on selected RDMA port; " - "set VGWRDMA_GID_INDEX to a valid RoCE GID index from `ibv_devinfo -v`"); + "no usable GID found on selected RoCE port; " + "set VGWRDMA_GID_INDEX to a valid GID index from `ibv_devinfo -v`"); return false; }