mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-22 07:54:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: port the cuObjServer wrapper to libcuobjserver 2.0.0

Browse Source 
NVIDIA's rolling cuda-rhel9 repository bumped libcuobjserver from `1.2.0` to `2.0.0` and `build/vgwrdma-builder/Dockerfile` installs it unpinned, so `make vgwrdma-docker` broke: `setTelemFlags` gained a second mask and `initRDMAConfigParams` vanished along with the `RDMAConnection` base class that 2.0.0 deletes entirely. Rather than pin the package, this ports the C wrapper and the Go `rdma` package to the new API.

Telemetry now calls `setTelemFlags(flags, 0)`, where 0 reproduces the old behaviour. `Server.InitRDMAConfig` and its C entry point are removed, which drops a method from the exported API of `github.com/versity/versitygw/rdma`; it had no callers, and 2.0.0 only accepts tunables through the four-argument constructor that `NewServer` already uses. The `dlsym` lookups for `startRDMASession` and `closeRDMASession` go too, as dead code: 1.2.0 never exported those symbols either, so the fallback paths were always what ran. `HandleGet` and `HandlePut` now wrap the library's negative return in a `syscall.Errno` so 2.0.0's new `-EPROTO` is legible, and the wrapper compile rule gains `-std=c++17`. No defaults change.

`make vgwrdma-docker` passes and the binary links `libcuobjserver.so.2`, so it requires a 2.x install at runtime. `cuobjtest-gpu` and `cuobjtest-host` are unaffected. The RDMA data path itself is not verified here; that needs Mellanox hardware with DC transport.
This commit is contained in:
niksis02
2026-09-10 21:15:01 +04:00
parent be3ec8ba4d
commit ee25c95948
6 changed files with 91 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1 -1
View File
@@ -74,7 +74,7 @@ $(BIN):
$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
$(VGWRDMA_WRAPPER_LIB): cuwrapper/cuobjserver_wrapper.cpp cuwrapper/cuobjserver_wrapper.h
$(CXX) -c -fPIC \
$(CXX) -c -fPIC -std=c++17 \
-I$(CUOBJ_SERVER_INC_DIR) -Icuwrapper \
-o cuwrapper/cuobjserver_wrapper.o \
cuwrapper/cuobjserver_wrapper.cpp
cubackend/cuserver.go
+4 -4
View File
@@ -59,10 +59,10 @@ func New(opts CuServerOpts, be backend.Backend) (*CuServer, error) {
return nil, fmt.Errorf("cuserver: rdma server: %w", err)
}
// StartSession is a no-op when the library manages session start
// internally (e.g. libcuobjserver v1.2.0 calls startRDMASession from
// the cuObjServer constructor). It is kept here for forward compatibility
// with library versions that require an explicit call.
// libcuobjserver starts the RDMA session inside the cuObjServer
// constructor, so this verifies the session came up rather than
// initiating it, and fails fast if the server constructed but did not
// connect.
if err := rdmaSrv.StartSession(); err != nil {
rdmaSrv.Close()
be.Shutdown()
cuwrapper/cuobjserver_wrapper.cpp
+23 -72
View File
@@ -18,7 +18,6 @@
#include "cuobjserver_wrapper.h"
#include "cuobjserver.h"
#include <dlfcn.h>
#include <stdio.h>
#include <stdlib.h>
#include <atomic>
@@ -26,54 +25,24 @@
#include <string>
// ---------------------------------------------------------------------------
// Session management — runtime symbol resolution
// Session management
//
// libcuobjserver.so may export startRDMASession / closeRDMASession under one
// of two C++ mangled names depending on when the library was compiled:
// libcuobjserver does not expose an explicit RDMA session start/close entry
// point. cuObjServer's constructor brings the session up and its destructor
// tears it down.
//
// Newer header layout (RDMAConnection base class):
// _ZN14RDMAConnection16startRDMASessionEv
// _ZN14RDMAConnection16closeRDMASessionEv
//
// Older layout (method directly on cuObjServer):
// _ZN11cuObjServer16startRDMASessionEv
// _ZN11cuObjServer16closeRDMASessionEv
//
// We resolve lazily at first call so neither name needs to be present at
// link time, and the code works with both library generations.
// Earlier revisions of this wrapper resolved startRDMASession/closeRDMASession
// at runtime via dlsym, trying both the RDMAConnection and cuObjServer mangled
// names. Neither symbol was ever exported, so those lookups always failed and
// the fallback paths below are what has always actually run. cuObjServer 2.0.0
// deletes the RDMAConnection base class outright, so the lookup can no longer
// succeed even in principle — it is gone rather than left as dead code.
// ---------------------------------------------------------------------------
typedef int (*rdma_start_fn_t)(void *);
typedef void (*rdma_close_fn_t)(void *);
// Verbose wrapper logs are enabled when cuobj_server_set_telem_flags includes
// info/debug bits (configured by cuserver -debug).
static std::atomic<bool> g_verbose_logs{false};
static rdma_start_fn_t find_start_rdma_session() {
void *proc = dlopen(nullptr, RTLD_LAZY);
if (!proc) return nullptr;
rdma_start_fn_t fn = reinterpret_cast<rdma_start_fn_t>(
dlsym(proc, "_ZN14RDMAConnection16startRDMASessionEv"));
if (!fn)
fn = reinterpret_cast<rdma_start_fn_t>(
dlsym(proc, "_ZN11cuObjServer16startRDMASessionEv"));
dlclose(proc);
return fn;
}
static rdma_close_fn_t find_close_rdma_session() {
void *proc = dlopen(nullptr, RTLD_LAZY);
if (!proc) return nullptr;
rdma_close_fn_t fn = reinterpret_cast<rdma_close_fn_t>(
dlsym(proc, "_ZN14RDMAConnection16closeRDMASessionEv"));
if (!fn)
fn = reinterpret_cast<rdma_close_fn_t>(
dlsym(proc, "_ZN11cuObjServer16closeRDMASessionEv"));
dlclose(proc);
return fn;
}
extern "C" {
cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigned proto) {
@@ -85,8 +54,8 @@ cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigne
}
}
// Build a cuObjRDMATunable from the flat C struct, shared by
// cuobj_server_create_with_config and cuobj_server_init_rdma_config.
// Build a cuObjRDMATunable from the flat C struct for
// cuobj_server_create_with_config.
static cuObjRDMATunable tunables_from_c(const cuobj_rdma_tunables_t *t) {
cuObjRDMATunable config;
config.setNumDcis(t->num_dcis);
@@ -122,27 +91,16 @@ void cuobj_server_destroy(cuobj_server_t *srv) {
}
int cuobj_server_start_session(cuobj_server_t *srv) {
static rdma_start_fn_t fn = find_start_rdma_session();
if (!fn) {
// Symbol not exported — library calls startRDMASession() internally
// from the cuObjServer constructor. Verify the session actually came
// up instead of unconditionally reporting success.
auto *s = reinterpret_cast<cuObjServer*>(srv);
return s->isConnected() ? 0 : -1;
}
int rc = fn(reinterpret_cast<void *>(srv));
if (rc != 0)
fprintf(stderr, "cuobjwrapper: startRDMASession returned %d\n", rc);
return rc;
// The cuObjServer constructor starts the session. Verify it actually came
// up instead of unconditionally reporting success.
auto *s = reinterpret_cast<cuObjServer*>(srv);
return s->isConnected() ? 0 : -1;
}
void cuobj_server_close_session(cuobj_server_t *srv) {
static rdma_close_fn_t fn = find_close_rdma_session();
if (!fn) {
// Symbol not exported — session cleanup handled by destructor.
return;
}
fn(reinterpret_cast<void *>(srv));
// Session teardown is owned by ~cuObjServer(); there is no separate close
// entry point. cuobj_server_destroy() is what actually closes the session.
(void)srv;
}
int cuobj_server_is_connected(cuobj_server_t *srv) {
@@ -231,17 +189,10 @@ void cuobj_server_shutdown_telemetry(void) {
void cuobj_server_set_telem_flags(unsigned flags) {
g_verbose_logs.store((flags & 0x0003u) != 0u);
cuObjServer::setTelemFlags(flags);
}
int cuobj_server_init_rdma_config(cuobj_server_t *srv, const cuobj_rdma_tunables_t *t) {
auto *s = reinterpret_cast<cuObjServer*>(srv);
try {
s->initRDMAConfigParams(tunables_from_c(t));
return 0;
} catch (...) {
return -1;
}
// 2.0.0 added a second mask selecting which operations get logged
// (CUOBJ_LOG_OP_GET / CUOBJ_LOG_OP_PUT). cuObjTelem initialises it to 0,
// so passing 0 keeps the 1.2.0 behaviour of no per-operation logging.
cuObjServer::setTelemFlags(flags, 0);
}
} // extern "C"
cuwrapper/cuobjserver_wrapper.h
+28 -9
View File
@@ -39,6 +39,13 @@ cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigne
void cuobj_server_destroy(cuobj_server_t *srv);
// RDMA session
//
// libcuobjserver has never exported an explicit session start/close entry
// point: the constructor brings the session up and the destructor tears it
// down. cuObjServer 2.0.0 makes that official by deleting the RDMAConnection
// base class that once declared start/closeRDMASession(). These two calls are
// kept so callers can express lifecycle intent — start_session reports whether
// the constructor-started session actually came up, close_session is a no-op.
int cuobj_server_start_session(cuobj_server_t *srv);
void cuobj_server_close_session(cuobj_server_t *srv);
int cuobj_server_is_connected(cuobj_server_t *srv);
@@ -58,7 +65,9 @@ void cuobj_server_free_channel(cuobj_server_t *srv, uint16_t channel_id);
// Data transfer (synchronous, no poll_delay override)
//
// handleGetObject: RDMA WRITE server→client (serves a GET request)
// Returns bytes transferred or -1 on error.
// Returns bytes transferred, or a negative errno on failure
// (-EPROTO when the RDMA descriptor is malformed or its prefix does not
// match the server's protocol).
ssize_t cuobj_server_handle_get(cuobj_server_t *srv,
const char *key,
cuobj_rdma_buffer_t *local_buf,
@@ -68,7 +77,9 @@ ssize_t cuobj_server_handle_get(cuobj_server_t *srv,
uint16_t channel);
// handlePutObject: RDMA READ client→server (serves a PUT request)
// Returns bytes transferred or -1 on error.
// Returns bytes transferred, or a negative errno on failure
// (-EPROTO when the RDMA descriptor is malformed or its prefix does not
// match the server's protocol).
ssize_t cuobj_server_handle_put(cuobj_server_t *srv,
const char *key,
cuobj_rdma_buffer_t *local_buf,
@@ -78,12 +89,23 @@ ssize_t cuobj_server_handle_put(cuobj_server_t *srv,
uint16_t channel);
// Telemetry (optional)
//
// cuObjServer 2.0.0 split setTelemFlags into (log_flags, log_op_flags), where
// the second mask (CUOBJ_LOG_OP_GET / CUOBJ_LOG_OP_PUT) enables per-operation
// logging. That mask is not exposed here: cuObjTelem defaults it to 0, so
// passing 0 reproduces the 1.2.0 behaviour this wrapper was written against.
void cuobj_server_setup_telemetry(int use_otel);
void cuobj_server_shutdown_telemetry(void);
void cuobj_server_set_telem_flags(unsigned flags);
// RDMA tunable parameters — flat C struct for CGO compatibility.
// Field names and defaults match cuObjRDMATunableParam in cuobjrdma.h.
//
// cuObjServer 2.0.0 adds two further tunables, deliberately omitted here so
// they keep their library defaults: `proto` (already defaults to
// CUOBJ_PROTO_RDMA_DC_V1, the only protocol this gateway implements) and
// `drain_wait_ms` (only consulted by the multi-VIP removeVip path, which this
// wrapper does not expose).
typedef struct {
int num_dcis; // default 128
unsigned cq_depth; // default 640
@@ -100,14 +122,11 @@ typedef struct {
int max_rd_atomic; // default 0 (auto)
} cuobj_rdma_tunables_t;
// Apply RDMA tuning parameters to an existing connection object.
// Takes effect on the next reconnection if called after session start.
// Returns 0 on success, -1 on error.
int cuobj_server_init_rdma_config(cuobj_server_t *srv, const cuobj_rdma_tunables_t *t);
// Create a cuObjServer with tunable parameters applied before the session
// starts. This is the preferred constructor when non-default tunables are
// needed, since the library starts the RDMA session inside the constructor.
// starts. This is the only way to set tunables: cuObjServer 2.0.0 deleted the
// RDMAConnection base class, and with it initRDMAConfigParams(), so tunables
// can no longer be applied to an already-constructed server. The library
// starts the RDMA session inside the constructor regardless.
cuobj_server_t* cuobj_server_create_with_config(const char *ip, unsigned short port, unsigned proto, const cuobj_rdma_tunables_t *t);
#ifdef __cplusplus
rdma/rdma.go
+35 -18
View File
@@ -19,7 +19,9 @@ package rdma
/*
#cgo CFLAGS: -I${SRCDIR}/../include -I${SRCDIR}/../cuwrapper
#cgo LDFLAGS: -L${SRCDIR} -l:libcuobjwrapper.a -L/usr/lib64 -lcuobjserver -lstdc++ -ldl
// Links against cuObjServer 2.x: the resulting binary records a
// libcuobjserver.so.2 NEEDED entry and will not run against a 1.x install.
#cgo LDFLAGS: -L${SRCDIR} -l:libcuobjwrapper.a -L/usr/lib64 -lcuobjserver -lstdc++
#include "cuobjserver_wrapper.h"
#include <stdlib.h>
*/
@@ -30,6 +32,7 @@ import (
"fmt"
"sync"
"sync/atomic"
"syscall"
"unsafe"
)
@@ -112,6 +115,10 @@ func tunablesToC(t RDMATunables) C.cuobj_rdma_tunables_t {
// Uses CUOBJ_PROTO_RDMA_DC_V1 (1001). If tunables is non-nil, the 4-argument
// constructor is used so the tunable parameters apply to the initial session
// started by the constructor. Pass nil to use library defaults.
//
// The constructor is the only point at which tunables can be applied:
// libcuobjserver starts the RDMA session inside it, and cuObjServer 2.0.0
// removed the after-the-fact initRDMAConfigParams() entry point.
func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error) {
cip := C.CString(ip)
defer C.free(unsafe.Pointer(cip))
@@ -127,16 +134,21 @@ func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error)
return nil, fmt.Errorf("rdma: failed to create cuObjServer on %s:%d", ip, port)
}
srv := &Server{csrv: csrv}
// Some library versions start the session as part of construction;
// record that readiness so StartSession can be a no-op and Close/CloseSession
// use a consistent ownership model.
// The library starts the session as part of construction; record that
// readiness so StartSession can be a no-op and Close/CloseSession use a
// consistent ownership model.
if srv.IsConnected() {
srv.sessionOpen = true
}
return srv, nil
}
// StartSession initiates the RDMA listening session.
// StartSession reports whether the RDMA session is up.
//
// libcuobjserver starts the session inside the cuObjServer constructor and has
// never exposed an explicit start entry point, so this verifies the session
// came up rather than initiating it. It is kept as an explicit lifecycle step
// so callers can fail fast on a server that constructed but did not connect.
// StartSession must not be called concurrently with CloseSession or Close.
func (s *Server) StartSession() error {
s.mu.Lock()
@@ -153,7 +165,7 @@ func (s *Server) StartSession() error {
s.mu.Unlock()
return nil
}
return fmt.Errorf("rdma: startRDMASession failed (rc=%d)", rc)
return fmt.Errorf("rdma: RDMA session is not connected (rc=%d)", rc)
}
s.mu.Lock()
s.sessionOpen = true
@@ -161,15 +173,6 @@ func (s *Server) StartSession() error {
return nil
}
// InitRDMAConfig applies RDMA tuning parameters. Must be called before StartSession.
func (s *Server) InitRDMAConfig(t RDMATunables) error {
ct := tunablesToC(t)
if rc := C.cuobj_server_init_rdma_config(s.csrv, &ct); rc != 0 {
return fmt.Errorf("rdma: initRDMAConfigParams failed (rc=%d)", rc)
}
return nil
}
// IsConnected returns the RDMA connection status.
func (s *Server) IsConnected() bool {
return C.cuobj_server_is_connected(s.csrv) != 0
@@ -236,6 +239,12 @@ func (s *Server) FreeChannel(id uint16) {
// HandleGet performs an RDMA WRITE (server→client) to serve a GET request.
// The local buffer must already contain the data to send.
// Returns bytes transferred.
//
// On failure the library returns a negative errno, which is wrapped into the
// returned error: a malformed RDMA descriptor, or one whose prefix does not
// match the server's protocol, reports syscall.EPROTO and can be matched with
// errors.Is. That distinguishes a bad client-supplied descriptor from a
// transport fault.
func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int64, rdmaDescr string, channel uint16) (int64, error) {
if buf == nil || buf.cbuf == nil {
return 0, errors.New("rdma: invalid or deregistered buffer")
@@ -255,7 +264,7 @@ func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int
n := C.cuobj_server_handle_get(s.csrv, ckey, buf.cbuf,
C.uint64_t(remoteStart), C.size_t(size), cdescr, C.uint16_t(channel))
if n < 0 {
return 0, fmt.Errorf("rdma: handleGetObject failed (rc=%d)", n)
return 0, fmt.Errorf("rdma: handleGetObject failed (rc=%d): %w", n, syscall.Errno(-int64(n)))
}
return int64(n), nil
}
@@ -263,6 +272,12 @@ func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int
// HandlePut performs an RDMA READ (client→server) to serve a PUT request.
// After return, the local buffer contains the data read from the client.
// Returns bytes transferred.
//
// On failure the library returns a negative errno, which is wrapped into the
// returned error: a malformed RDMA descriptor, or one whose prefix does not
// match the server's protocol, reports syscall.EPROTO and can be matched with
// errors.Is. That distinguishes a bad client-supplied descriptor from a
// transport fault.
func (s *Server) HandlePut(key string, buf *Buffer, remoteStart uint64, size int64, rdmaDescr string, channel uint16) (int64, error) {
if buf == nil || buf.cbuf == nil {
return 0, errors.New("rdma: invalid or deregistered buffer")
@@ -282,12 +297,14 @@ func (s *Server) HandlePut(key string, buf *Buffer, remoteStart uint64, size int
n := C.cuobj_server_handle_put(s.csrv, ckey, buf.cbuf,
C.uint64_t(remoteStart), C.size_t(size), cdescr, C.uint16_t(channel))
if n < 0 {
return 0, fmt.Errorf("rdma: handlePutObject failed (rc=%d)", n)
return 0, fmt.Errorf("rdma: handlePutObject failed (rc=%d): %w", n, syscall.Errno(-int64(n)))
}
return int64(n), nil
}
// CloseSession tears down the RDMA session.
// CloseSession marks the session closed. The underlying library ties session
// teardown to the cuObjServer destructor, so the session is not actually torn
// down until Close is called.
func (s *Server) CloseSession() {
s.mu.Lock()
defer s.mu.Unlock()
rdma/rdma_stub.go
-3
View File
@@ -57,9 +57,6 @@ func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error)
// StartSession always returns errNotSupported on this platform.
func (s *Server) StartSession() error { return errNotSupported }
// InitRDMAConfig always returns errNotSupported on this platform.
func (s *Server) InitRDMAConfig(t RDMATunables) error { return errNotSupported }
// IsConnected always returns false on this platform.
func (s *Server) IsConnected() bool { return false }