mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-10-22 06:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

rdma: add --rc-device to pick the RC data plane's verbs device by name

Browse Source 
The hipobj-rc-v2 server chooses its HCA as "the first verbs device
that opens", or the first whose GID begins with --rc-gid-hint. Neither
works on a storage node with more than one RDMA adapter. Enumeration
order is not a fabric choice, and the hint compares only the first
four GID bytes: every port's GID 0 is link-local, so an InfiniBand
HCA and a RoCE NIC on the same host both match fe:80:0:0, and IPv4-
mapped RoCE GIDs all match 0:0:0:0.

We hit this on a node with two ConnectX-6 IB ports and two Broadcom
RoCE ports: the gateway silently bound bnxt_re0 (link_layer Ethernet,
LID 0, no path to the clients on the IB fabric) and every RC session
timed out in the data phase with nothing in the log to say why.

Add --rc-device / VGW_RC_DEVICE, plumbed through DeviceOpts.DevName
and rc_device_opts.dev_name, that filters the device list by
ibv_device.name before the GID-hint match. It takes precedence over
the hint; the hint still applies within the named device. The failure
message now names the device that was asked for.
This commit is contained in:
Kyeongpyo Kim
2026-09-27 02:34:26 +09:00
parent f7a19e1fd3
commit 50b098c68c
5 changed files with 41 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/vgwrdma/main.go
+8
View File
@@ -115,6 +115,7 @@ var (
socketPerm string
rdmaIP string
rcGidHint string
rcDevice string
rdmaRCEnable bool
rdmaPort uint
poolBufSize int
@@ -905,6 +906,12 @@ func initFlags() []cli.Flag {
EnvVars: []string{"VGW_RDMA_IP"},
Destination: &rdmaIP,
},
&cli.StringFlag{
Name: "rc-device",
Usage: "verbs device name (e.g. mlx5_0) for the hipobj-rc-v2 RC data plane; takes precedence over --rc-gid-hint",
EnvVars: []string{"VGW_RC_DEVICE"},
Destination: &rcDevice,
},
&cli.StringFlag{
Name: "rc-gid-hint",
Usage: "dotted GID prefix selecting the verbs device for the hipobj-rc-v2 RC data plane (default: first device)",
@@ -1264,6 +1271,7 @@ func runGateway(ctx context.Context, be backend.Backend) error {
rcSvc, err := rcserver.Init(rcserver.DeviceOpts{
GidHint: rcGidHint,
DevName: rcDevice,
Port: 1,
MaxSessions: uint32(rcMaxSessions),
MaxUserSessions: uint32(rcMaxUserSessions),
cuwrapper/rc/rc_server_abi.cpp
+17 -5
View File
@@ -346,11 +346,17 @@ int rc_server_init(const rc_device_opts *opts, rc_server **out) {
return RC_E_INTERNAL;
}
struct ibv_device *chosen = devs[0];
/* GID hint: pick the first device/port whose GID starts with it.
* Query with srv->opts.port, which the normalization above has
* already made 1-based. */
/* Device name, when given, is the primary filter: on a host with
* several HCAs (an IB adapter next to a RoCE NIC, say) neither
* "first device" nor the 4-byte GID prefix can single out the
* one on the right fabric, since every link-local GID starts
* fe:80:0:0. Then the GID hint: pick the first device/port whose
* GID starts with it. Query with srv->opts.port, which the
* normalization above has already made 1-based. */
const bool byName = opts->dev_name && *opts->dev_name;
struct ibv_context *ctx = nullptr;
for (int i = 0; i < n && !ctx; i++) {
if (byName && strcmp(devs[i]->name, opts->dev_name) != 0) continue;
struct ibv_context *c = hipObj::ibv.open_device(devs[i]);
if (!c) continue;
if (opts->gid_hint) {
@@ -378,8 +384,14 @@ int rc_server_init(const rc_device_opts *opts, rc_server **out) {
}
if (!ctx) {
hipObj::ibv.free_device_list(devs);
fprintf(stderr, "rc: no verbs device matches gid_hint %.32s\n",
opts->gid_hint ? opts->gid_hint : "");
if (byName) {
fprintf(stderr, "rc: no verbs device named %.64s%s\n",
opts->dev_name,
opts->gid_hint ? " with a GID matching gid_hint" : "");
} else {
fprintf(stderr, "rc: no verbs device matches gid_hint %.32s\n",
opts->gid_hint ? opts->gid_hint : "");
}
return RC_E_INTERNAL;
}
struct ibv_pd *pd = hipObj::ibv.alloc_pd(ctx);
cuwrapper/rc/rc_server_abi.h
+4 -2
View File
@@ -67,10 +67,12 @@ typedef void (*rc_terminal_fn)(void *ctx, const char *id, int outcome,
void rc_server_set_terminal_notify(rc_server *srv, rc_terminal_fn fn,
void *ctx);
/* Device selection: matching GID prefix when gid_hint is set,
* otherwise the first verbs device. */
/* Device selection: the verbs device named by dev_name when set,
* else the first device whose GID starts with gid_hint when that is
* set, otherwise the first verbs device that opens. */
typedef struct {
const char *gid_hint; /* nullable; dotted GID prefix */
const char *dev_name; /* nullable; verbs device name, e.g. mlx5_0 */
uint8_t port;
int gid_index;
uint32_t max_sessions;
rdma/rcserver/rcserver_linux.go
+11
View File
@@ -71,6 +71,11 @@ type DeviceOpts struct {
// GidHint selects the device whose GID starts with this dotted
// prefix; empty picks the first verbs device.
GidHint string
// DevName selects the verbs device by name (e.g. mlx5_0). It takes
// precedence over GidHint and the first-device default, which
// cannot tell HCAs apart on a host where every port's GID 0 is
// link-local.
DevName string
Port uint8
GidIdx int
// Global limits.
@@ -256,6 +261,12 @@ func Init(opts DeviceOpts) (*RCSvc, error) {
defer C.free(unsafe.Pointer(gidHint))
}
copts.gid_hint = gidHint
var devName *C.char
if opts.DevName != "" {
devName = C.CString(opts.DevName)
defer C.free(unsafe.Pointer(devName))
}
copts.dev_name = devName
var srv *C.rc_server
if rc := C.rc_server_init(&copts, &srv); rc != C.RC_OK {
rdma/rcserver/rcserver_stub.go
+1
View File
@@ -25,6 +25,7 @@ var errNotSupported = errors.New("rcserver: not supported on this platform")
// DeviceOpts configures device selection and resource limits.
type DeviceOpts struct {
GidHint string
DevName string
Port uint8
GidIdx int
MaxSessions uint32