mirror of
https://github.com/versity/versitygw.git
synced 2026-09-29 19:25:40 +00:00
fix: port the cuObjServer wrapper to libcuobjserver 2.0.0
NVIDIA's rolling cuda-rhel9 repository bumped libcuobjserver from `1.2.0` to `2.0.0` and `build/vgwrdma-builder/Dockerfile` installs it unpinned, so `make vgwrdma-docker` broke: `setTelemFlags` gained a second mask and `initRDMAConfigParams` vanished along with the `RDMAConnection` base class that 2.0.0 deletes entirely. Rather than pin the package, this ports the C wrapper and the Go `rdma` package to the new API. Telemetry now calls `setTelemFlags(flags, 0)`, where 0 reproduces the old behaviour. `Server.InitRDMAConfig` and its C entry point are removed, which drops a method from the exported API of `github.com/versity/versitygw/rdma`; it had no callers, and 2.0.0 only accepts tunables through the four-argument constructor that `NewServer` already uses. The `dlsym` lookups for `startRDMASession` and `closeRDMASession` go too, as dead code: 1.2.0 never exported those symbols either, so the fallback paths were always what ran. `HandleGet` and `HandlePut` now wrap the library's negative return in a `syscall.Errno` so 2.0.0's new `-EPROTO` is legible, and the wrapper compile rule gains `-std=c++17`. No defaults change. `make vgwrdma-docker` passes and the binary links `libcuobjserver.so.2`, so it requires a 2.x install at runtime. `cuobjtest-gpu` and `cuobjtest-host` are unaffected. The RDMA data path itself is not verified here; that needs Mellanox hardware with DC transport.
This commit is contained in:
@@ -18,7 +18,6 @@
|
||||
#include "cuobjserver_wrapper.h"
|
||||
#include "cuobjserver.h"
|
||||
|
||||
#include <dlfcn.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <atomic>
|
||||
@@ -26,54 +25,24 @@
|
||||
#include <string>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Session management — runtime symbol resolution
|
||||
// Session management
|
||||
//
|
||||
// libcuobjserver.so may export startRDMASession / closeRDMASession under one
|
||||
// of two C++ mangled names depending on when the library was compiled:
|
||||
// libcuobjserver does not expose an explicit RDMA session start/close entry
|
||||
// point. cuObjServer's constructor brings the session up and its destructor
|
||||
// tears it down.
|
||||
//
|
||||
// Newer header layout (RDMAConnection base class):
|
||||
// _ZN14RDMAConnection16startRDMASessionEv
|
||||
// _ZN14RDMAConnection16closeRDMASessionEv
|
||||
//
|
||||
// Older layout (method directly on cuObjServer):
|
||||
// _ZN11cuObjServer16startRDMASessionEv
|
||||
// _ZN11cuObjServer16closeRDMASessionEv
|
||||
//
|
||||
// We resolve lazily at first call so neither name needs to be present at
|
||||
// link time, and the code works with both library generations.
|
||||
// Earlier revisions of this wrapper resolved startRDMASession/closeRDMASession
|
||||
// at runtime via dlsym, trying both the RDMAConnection and cuObjServer mangled
|
||||
// names. Neither symbol was ever exported, so those lookups always failed and
|
||||
// the fallback paths below are what has always actually run. cuObjServer 2.0.0
|
||||
// deletes the RDMAConnection base class outright, so the lookup can no longer
|
||||
// succeed even in principle — it is gone rather than left as dead code.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
typedef int (*rdma_start_fn_t)(void *);
|
||||
typedef void (*rdma_close_fn_t)(void *);
|
||||
|
||||
// Verbose wrapper logs are enabled when cuobj_server_set_telem_flags includes
|
||||
// info/debug bits (configured by cuserver -debug).
|
||||
static std::atomic<bool> g_verbose_logs{false};
|
||||
|
||||
static rdma_start_fn_t find_start_rdma_session() {
|
||||
void *proc = dlopen(nullptr, RTLD_LAZY);
|
||||
if (!proc) return nullptr;
|
||||
rdma_start_fn_t fn = reinterpret_cast<rdma_start_fn_t>(
|
||||
dlsym(proc, "_ZN14RDMAConnection16startRDMASessionEv"));
|
||||
if (!fn)
|
||||
fn = reinterpret_cast<rdma_start_fn_t>(
|
||||
dlsym(proc, "_ZN11cuObjServer16startRDMASessionEv"));
|
||||
dlclose(proc);
|
||||
return fn;
|
||||
}
|
||||
|
||||
static rdma_close_fn_t find_close_rdma_session() {
|
||||
void *proc = dlopen(nullptr, RTLD_LAZY);
|
||||
if (!proc) return nullptr;
|
||||
rdma_close_fn_t fn = reinterpret_cast<rdma_close_fn_t>(
|
||||
dlsym(proc, "_ZN14RDMAConnection16closeRDMASessionEv"));
|
||||
if (!fn)
|
||||
fn = reinterpret_cast<rdma_close_fn_t>(
|
||||
dlsym(proc, "_ZN11cuObjServer16closeRDMASessionEv"));
|
||||
dlclose(proc);
|
||||
return fn;
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
|
||||
cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigned proto) {
|
||||
@@ -85,8 +54,8 @@ cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigne
|
||||
}
|
||||
}
|
||||
|
||||
// Build a cuObjRDMATunable from the flat C struct, shared by
|
||||
// cuobj_server_create_with_config and cuobj_server_init_rdma_config.
|
||||
// Build a cuObjRDMATunable from the flat C struct for
|
||||
// cuobj_server_create_with_config.
|
||||
static cuObjRDMATunable tunables_from_c(const cuobj_rdma_tunables_t *t) {
|
||||
cuObjRDMATunable config;
|
||||
config.setNumDcis(t->num_dcis);
|
||||
@@ -122,27 +91,16 @@ void cuobj_server_destroy(cuobj_server_t *srv) {
|
||||
}
|
||||
|
||||
int cuobj_server_start_session(cuobj_server_t *srv) {
|
||||
static rdma_start_fn_t fn = find_start_rdma_session();
|
||||
if (!fn) {
|
||||
// Symbol not exported — library calls startRDMASession() internally
|
||||
// from the cuObjServer constructor. Verify the session actually came
|
||||
// up instead of unconditionally reporting success.
|
||||
auto *s = reinterpret_cast<cuObjServer*>(srv);
|
||||
return s->isConnected() ? 0 : -1;
|
||||
}
|
||||
int rc = fn(reinterpret_cast<void *>(srv));
|
||||
if (rc != 0)
|
||||
fprintf(stderr, "cuobjwrapper: startRDMASession returned %d\n", rc);
|
||||
return rc;
|
||||
// The cuObjServer constructor starts the session. Verify it actually came
|
||||
// up instead of unconditionally reporting success.
|
||||
auto *s = reinterpret_cast<cuObjServer*>(srv);
|
||||
return s->isConnected() ? 0 : -1;
|
||||
}
|
||||
|
||||
void cuobj_server_close_session(cuobj_server_t *srv) {
|
||||
static rdma_close_fn_t fn = find_close_rdma_session();
|
||||
if (!fn) {
|
||||
// Symbol not exported — session cleanup handled by destructor.
|
||||
return;
|
||||
}
|
||||
fn(reinterpret_cast<void *>(srv));
|
||||
// Session teardown is owned by ~cuObjServer(); there is no separate close
|
||||
// entry point. cuobj_server_destroy() is what actually closes the session.
|
||||
(void)srv;
|
||||
}
|
||||
|
||||
int cuobj_server_is_connected(cuobj_server_t *srv) {
|
||||
@@ -231,17 +189,10 @@ void cuobj_server_shutdown_telemetry(void) {
|
||||
|
||||
void cuobj_server_set_telem_flags(unsigned flags) {
|
||||
g_verbose_logs.store((flags & 0x0003u) != 0u);
|
||||
cuObjServer::setTelemFlags(flags);
|
||||
}
|
||||
|
||||
int cuobj_server_init_rdma_config(cuobj_server_t *srv, const cuobj_rdma_tunables_t *t) {
|
||||
auto *s = reinterpret_cast<cuObjServer*>(srv);
|
||||
try {
|
||||
s->initRDMAConfigParams(tunables_from_c(t));
|
||||
return 0;
|
||||
} catch (...) {
|
||||
return -1;
|
||||
}
|
||||
// 2.0.0 added a second mask selecting which operations get logged
|
||||
// (CUOBJ_LOG_OP_GET / CUOBJ_LOG_OP_PUT). cuObjTelem initialises it to 0,
|
||||
// so passing 0 keeps the 1.2.0 behaviour of no per-operation logging.
|
||||
cuObjServer::setTelemFlags(flags, 0);
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
|
||||
Reference in New Issue
Block a user