From ee25c9594882c5cac3cf71f1cdce34cd918230ce Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 10 Sep 2026 21:15:01 +0400 Subject: [PATCH] fix: port the cuObjServer wrapper to libcuobjserver 2.0.0 NVIDIA's rolling cuda-rhel9 repository bumped libcuobjserver from `1.2.0` to `2.0.0` and `build/vgwrdma-builder/Dockerfile` installs it unpinned, so `make vgwrdma-docker` broke: `setTelemFlags` gained a second mask and `initRDMAConfigParams` vanished along with the `RDMAConnection` base class that 2.0.0 deletes entirely. Rather than pin the package, this ports the C wrapper and the Go `rdma` package to the new API. Telemetry now calls `setTelemFlags(flags, 0)`, where 0 reproduces the old behaviour. `Server.InitRDMAConfig` and its C entry point are removed, which drops a method from the exported API of `github.com/versity/versitygw/rdma`; it had no callers, and 2.0.0 only accepts tunables through the four-argument constructor that `NewServer` already uses. The `dlsym` lookups for `startRDMASession` and `closeRDMASession` go too, as dead code: 1.2.0 never exported those symbols either, so the fallback paths were always what ran. `HandleGet` and `HandlePut` now wrap the library's negative return in a `syscall.Errno` so 2.0.0's new `-EPROTO` is legible, and the wrapper compile rule gains `-std=c++17`. No defaults change. `make vgwrdma-docker` passes and the binary links `libcuobjserver.so.2`, so it requires a 2.x install at runtime. `cuobjtest-gpu` and `cuobjtest-host` are unaffected. The RDMA data path itself is not verified here; that needs Mellanox hardware with DC transport. --- Makefile | 2 +- cubackend/cuserver.go | 8 +-- cuwrapper/cuobjserver_wrapper.cpp | 95 ++++++++----------------------- cuwrapper/cuobjserver_wrapper.h | 37 +++++++++--- rdma/rdma.go | 53 +++++++++++------ rdma/rdma_stub.go | 3 - 6 files changed, 91 insertions(+), 107 deletions(-) diff --git a/Makefile b/Makefile index 2f8f9bb8..5eed900e 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ $(BIN): $(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go $(VGWRDMA_WRAPPER_LIB): cuwrapper/cuobjserver_wrapper.cpp cuwrapper/cuobjserver_wrapper.h - $(CXX) -c -fPIC \ + $(CXX) -c -fPIC -std=c++17 \ -I$(CUOBJ_SERVER_INC_DIR) -Icuwrapper \ -o cuwrapper/cuobjserver_wrapper.o \ cuwrapper/cuobjserver_wrapper.cpp diff --git a/cubackend/cuserver.go b/cubackend/cuserver.go index 0e13ed21..0728d1a3 100644 --- a/cubackend/cuserver.go +++ b/cubackend/cuserver.go @@ -59,10 +59,10 @@ func New(opts CuServerOpts, be backend.Backend) (*CuServer, error) { return nil, fmt.Errorf("cuserver: rdma server: %w", err) } - // StartSession is a no-op when the library manages session start - // internally (e.g. libcuobjserver v1.2.0 calls startRDMASession from - // the cuObjServer constructor). It is kept here for forward compatibility - // with library versions that require an explicit call. + // libcuobjserver starts the RDMA session inside the cuObjServer + // constructor, so this verifies the session came up rather than + // initiating it, and fails fast if the server constructed but did not + // connect. if err := rdmaSrv.StartSession(); err != nil { rdmaSrv.Close() be.Shutdown() diff --git a/cuwrapper/cuobjserver_wrapper.cpp b/cuwrapper/cuobjserver_wrapper.cpp index 713d55e7..7b18da54 100644 --- a/cuwrapper/cuobjserver_wrapper.cpp +++ b/cuwrapper/cuobjserver_wrapper.cpp @@ -18,7 +18,6 @@ #include "cuobjserver_wrapper.h" #include "cuobjserver.h" -#include #include #include #include @@ -26,54 +25,24 @@ #include // --------------------------------------------------------------------------- -// Session management — runtime symbol resolution +// Session management // -// libcuobjserver.so may export startRDMASession / closeRDMASession under one -// of two C++ mangled names depending on when the library was compiled: +// libcuobjserver does not expose an explicit RDMA session start/close entry +// point. cuObjServer's constructor brings the session up and its destructor +// tears it down. // -// Newer header layout (RDMAConnection base class): -// _ZN14RDMAConnection16startRDMASessionEv -// _ZN14RDMAConnection16closeRDMASessionEv -// -// Older layout (method directly on cuObjServer): -// _ZN11cuObjServer16startRDMASessionEv -// _ZN11cuObjServer16closeRDMASessionEv -// -// We resolve lazily at first call so neither name needs to be present at -// link time, and the code works with both library generations. +// Earlier revisions of this wrapper resolved startRDMASession/closeRDMASession +// at runtime via dlsym, trying both the RDMAConnection and cuObjServer mangled +// names. Neither symbol was ever exported, so those lookups always failed and +// the fallback paths below are what has always actually run. cuObjServer 2.0.0 +// deletes the RDMAConnection base class outright, so the lookup can no longer +// succeed even in principle — it is gone rather than left as dead code. // --------------------------------------------------------------------------- -typedef int (*rdma_start_fn_t)(void *); -typedef void (*rdma_close_fn_t)(void *); - // Verbose wrapper logs are enabled when cuobj_server_set_telem_flags includes // info/debug bits (configured by cuserver -debug). static std::atomic g_verbose_logs{false}; -static rdma_start_fn_t find_start_rdma_session() { - void *proc = dlopen(nullptr, RTLD_LAZY); - if (!proc) return nullptr; - rdma_start_fn_t fn = reinterpret_cast( - dlsym(proc, "_ZN14RDMAConnection16startRDMASessionEv")); - if (!fn) - fn = reinterpret_cast( - dlsym(proc, "_ZN11cuObjServer16startRDMASessionEv")); - dlclose(proc); - return fn; -} - -static rdma_close_fn_t find_close_rdma_session() { - void *proc = dlopen(nullptr, RTLD_LAZY); - if (!proc) return nullptr; - rdma_close_fn_t fn = reinterpret_cast( - dlsym(proc, "_ZN14RDMAConnection16closeRDMASessionEv")); - if (!fn) - fn = reinterpret_cast( - dlsym(proc, "_ZN11cuObjServer16closeRDMASessionEv")); - dlclose(proc); - return fn; -} - extern "C" { cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigned proto) { @@ -85,8 +54,8 @@ cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigne } } -// Build a cuObjRDMATunable from the flat C struct, shared by -// cuobj_server_create_with_config and cuobj_server_init_rdma_config. +// Build a cuObjRDMATunable from the flat C struct for +// cuobj_server_create_with_config. static cuObjRDMATunable tunables_from_c(const cuobj_rdma_tunables_t *t) { cuObjRDMATunable config; config.setNumDcis(t->num_dcis); @@ -122,27 +91,16 @@ void cuobj_server_destroy(cuobj_server_t *srv) { } int cuobj_server_start_session(cuobj_server_t *srv) { - static rdma_start_fn_t fn = find_start_rdma_session(); - if (!fn) { - // Symbol not exported — library calls startRDMASession() internally - // from the cuObjServer constructor. Verify the session actually came - // up instead of unconditionally reporting success. - auto *s = reinterpret_cast(srv); - return s->isConnected() ? 0 : -1; - } - int rc = fn(reinterpret_cast(srv)); - if (rc != 0) - fprintf(stderr, "cuobjwrapper: startRDMASession returned %d\n", rc); - return rc; + // The cuObjServer constructor starts the session. Verify it actually came + // up instead of unconditionally reporting success. + auto *s = reinterpret_cast(srv); + return s->isConnected() ? 0 : -1; } void cuobj_server_close_session(cuobj_server_t *srv) { - static rdma_close_fn_t fn = find_close_rdma_session(); - if (!fn) { - // Symbol not exported — session cleanup handled by destructor. - return; - } - fn(reinterpret_cast(srv)); + // Session teardown is owned by ~cuObjServer(); there is no separate close + // entry point. cuobj_server_destroy() is what actually closes the session. + (void)srv; } int cuobj_server_is_connected(cuobj_server_t *srv) { @@ -231,17 +189,10 @@ void cuobj_server_shutdown_telemetry(void) { void cuobj_server_set_telem_flags(unsigned flags) { g_verbose_logs.store((flags & 0x0003u) != 0u); - cuObjServer::setTelemFlags(flags); -} - -int cuobj_server_init_rdma_config(cuobj_server_t *srv, const cuobj_rdma_tunables_t *t) { - auto *s = reinterpret_cast(srv); - try { - s->initRDMAConfigParams(tunables_from_c(t)); - return 0; - } catch (...) { - return -1; - } + // 2.0.0 added a second mask selecting which operations get logged + // (CUOBJ_LOG_OP_GET / CUOBJ_LOG_OP_PUT). cuObjTelem initialises it to 0, + // so passing 0 keeps the 1.2.0 behaviour of no per-operation logging. + cuObjServer::setTelemFlags(flags, 0); } } // extern "C" diff --git a/cuwrapper/cuobjserver_wrapper.h b/cuwrapper/cuobjserver_wrapper.h index 227ecd24..70d8ed34 100644 --- a/cuwrapper/cuobjserver_wrapper.h +++ b/cuwrapper/cuobjserver_wrapper.h @@ -39,6 +39,13 @@ cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigne void cuobj_server_destroy(cuobj_server_t *srv); // RDMA session +// +// libcuobjserver has never exported an explicit session start/close entry +// point: the constructor brings the session up and the destructor tears it +// down. cuObjServer 2.0.0 makes that official by deleting the RDMAConnection +// base class that once declared start/closeRDMASession(). These two calls are +// kept so callers can express lifecycle intent — start_session reports whether +// the constructor-started session actually came up, close_session is a no-op. int cuobj_server_start_session(cuobj_server_t *srv); void cuobj_server_close_session(cuobj_server_t *srv); int cuobj_server_is_connected(cuobj_server_t *srv); @@ -58,7 +65,9 @@ void cuobj_server_free_channel(cuobj_server_t *srv, uint16_t channel_id); // Data transfer (synchronous, no poll_delay override) // // handleGetObject: RDMA WRITE server→client (serves a GET request) -// Returns bytes transferred or -1 on error. +// Returns bytes transferred, or a negative errno on failure +// (-EPROTO when the RDMA descriptor is malformed or its prefix does not +// match the server's protocol). ssize_t cuobj_server_handle_get(cuobj_server_t *srv, const char *key, cuobj_rdma_buffer_t *local_buf, @@ -68,7 +77,9 @@ ssize_t cuobj_server_handle_get(cuobj_server_t *srv, uint16_t channel); // handlePutObject: RDMA READ client→server (serves a PUT request) -// Returns bytes transferred or -1 on error. +// Returns bytes transferred, or a negative errno on failure +// (-EPROTO when the RDMA descriptor is malformed or its prefix does not +// match the server's protocol). ssize_t cuobj_server_handle_put(cuobj_server_t *srv, const char *key, cuobj_rdma_buffer_t *local_buf, @@ -78,12 +89,23 @@ ssize_t cuobj_server_handle_put(cuobj_server_t *srv, uint16_t channel); // Telemetry (optional) +// +// cuObjServer 2.0.0 split setTelemFlags into (log_flags, log_op_flags), where +// the second mask (CUOBJ_LOG_OP_GET / CUOBJ_LOG_OP_PUT) enables per-operation +// logging. That mask is not exposed here: cuObjTelem defaults it to 0, so +// passing 0 reproduces the 1.2.0 behaviour this wrapper was written against. void cuobj_server_setup_telemetry(int use_otel); void cuobj_server_shutdown_telemetry(void); void cuobj_server_set_telem_flags(unsigned flags); // RDMA tunable parameters — flat C struct for CGO compatibility. // Field names and defaults match cuObjRDMATunableParam in cuobjrdma.h. +// +// cuObjServer 2.0.0 adds two further tunables, deliberately omitted here so +// they keep their library defaults: `proto` (already defaults to +// CUOBJ_PROTO_RDMA_DC_V1, the only protocol this gateway implements) and +// `drain_wait_ms` (only consulted by the multi-VIP removeVip path, which this +// wrapper does not expose). typedef struct { int num_dcis; // default 128 unsigned cq_depth; // default 640 @@ -100,14 +122,11 @@ typedef struct { int max_rd_atomic; // default 0 (auto) } cuobj_rdma_tunables_t; -// Apply RDMA tuning parameters to an existing connection object. -// Takes effect on the next reconnection if called after session start. -// Returns 0 on success, -1 on error. -int cuobj_server_init_rdma_config(cuobj_server_t *srv, const cuobj_rdma_tunables_t *t); - // Create a cuObjServer with tunable parameters applied before the session -// starts. This is the preferred constructor when non-default tunables are -// needed, since the library starts the RDMA session inside the constructor. +// starts. This is the only way to set tunables: cuObjServer 2.0.0 deleted the +// RDMAConnection base class, and with it initRDMAConfigParams(), so tunables +// can no longer be applied to an already-constructed server. The library +// starts the RDMA session inside the constructor regardless. cuobj_server_t* cuobj_server_create_with_config(const char *ip, unsigned short port, unsigned proto, const cuobj_rdma_tunables_t *t); #ifdef __cplusplus diff --git a/rdma/rdma.go b/rdma/rdma.go index c300c333..07187bf1 100644 --- a/rdma/rdma.go +++ b/rdma/rdma.go @@ -19,7 +19,9 @@ package rdma /* #cgo CFLAGS: -I${SRCDIR}/../include -I${SRCDIR}/../cuwrapper -#cgo LDFLAGS: -L${SRCDIR} -l:libcuobjwrapper.a -L/usr/lib64 -lcuobjserver -lstdc++ -ldl +// Links against cuObjServer 2.x: the resulting binary records a +// libcuobjserver.so.2 NEEDED entry and will not run against a 1.x install. +#cgo LDFLAGS: -L${SRCDIR} -l:libcuobjwrapper.a -L/usr/lib64 -lcuobjserver -lstdc++ #include "cuobjserver_wrapper.h" #include */ @@ -30,6 +32,7 @@ import ( "fmt" "sync" "sync/atomic" + "syscall" "unsafe" ) @@ -112,6 +115,10 @@ func tunablesToC(t RDMATunables) C.cuobj_rdma_tunables_t { // Uses CUOBJ_PROTO_RDMA_DC_V1 (1001). If tunables is non-nil, the 4-argument // constructor is used so the tunable parameters apply to the initial session // started by the constructor. Pass nil to use library defaults. +// +// The constructor is the only point at which tunables can be applied: +// libcuobjserver starts the RDMA session inside it, and cuObjServer 2.0.0 +// removed the after-the-fact initRDMAConfigParams() entry point. func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error) { cip := C.CString(ip) defer C.free(unsafe.Pointer(cip)) @@ -127,16 +134,21 @@ func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error) return nil, fmt.Errorf("rdma: failed to create cuObjServer on %s:%d", ip, port) } srv := &Server{csrv: csrv} - // Some library versions start the session as part of construction; - // record that readiness so StartSession can be a no-op and Close/CloseSession - // use a consistent ownership model. + // The library starts the session as part of construction; record that + // readiness so StartSession can be a no-op and Close/CloseSession use a + // consistent ownership model. if srv.IsConnected() { srv.sessionOpen = true } return srv, nil } -// StartSession initiates the RDMA listening session. +// StartSession reports whether the RDMA session is up. +// +// libcuobjserver starts the session inside the cuObjServer constructor and has +// never exposed an explicit start entry point, so this verifies the session +// came up rather than initiating it. It is kept as an explicit lifecycle step +// so callers can fail fast on a server that constructed but did not connect. // StartSession must not be called concurrently with CloseSession or Close. func (s *Server) StartSession() error { s.mu.Lock() @@ -153,7 +165,7 @@ func (s *Server) StartSession() error { s.mu.Unlock() return nil } - return fmt.Errorf("rdma: startRDMASession failed (rc=%d)", rc) + return fmt.Errorf("rdma: RDMA session is not connected (rc=%d)", rc) } s.mu.Lock() s.sessionOpen = true @@ -161,15 +173,6 @@ func (s *Server) StartSession() error { return nil } -// InitRDMAConfig applies RDMA tuning parameters. Must be called before StartSession. -func (s *Server) InitRDMAConfig(t RDMATunables) error { - ct := tunablesToC(t) - if rc := C.cuobj_server_init_rdma_config(s.csrv, &ct); rc != 0 { - return fmt.Errorf("rdma: initRDMAConfigParams failed (rc=%d)", rc) - } - return nil -} - // IsConnected returns the RDMA connection status. func (s *Server) IsConnected() bool { return C.cuobj_server_is_connected(s.csrv) != 0 @@ -236,6 +239,12 @@ func (s *Server) FreeChannel(id uint16) { // HandleGet performs an RDMA WRITE (server→client) to serve a GET request. // The local buffer must already contain the data to send. // Returns bytes transferred. +// +// On failure the library returns a negative errno, which is wrapped into the +// returned error: a malformed RDMA descriptor, or one whose prefix does not +// match the server's protocol, reports syscall.EPROTO and can be matched with +// errors.Is. That distinguishes a bad client-supplied descriptor from a +// transport fault. func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int64, rdmaDescr string, channel uint16) (int64, error) { if buf == nil || buf.cbuf == nil { return 0, errors.New("rdma: invalid or deregistered buffer") @@ -255,7 +264,7 @@ func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int n := C.cuobj_server_handle_get(s.csrv, ckey, buf.cbuf, C.uint64_t(remoteStart), C.size_t(size), cdescr, C.uint16_t(channel)) if n < 0 { - return 0, fmt.Errorf("rdma: handleGetObject failed (rc=%d)", n) + return 0, fmt.Errorf("rdma: handleGetObject failed (rc=%d): %w", n, syscall.Errno(-int64(n))) } return int64(n), nil } @@ -263,6 +272,12 @@ func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int // HandlePut performs an RDMA READ (client→server) to serve a PUT request. // After return, the local buffer contains the data read from the client. // Returns bytes transferred. +// +// On failure the library returns a negative errno, which is wrapped into the +// returned error: a malformed RDMA descriptor, or one whose prefix does not +// match the server's protocol, reports syscall.EPROTO and can be matched with +// errors.Is. That distinguishes a bad client-supplied descriptor from a +// transport fault. func (s *Server) HandlePut(key string, buf *Buffer, remoteStart uint64, size int64, rdmaDescr string, channel uint16) (int64, error) { if buf == nil || buf.cbuf == nil { return 0, errors.New("rdma: invalid or deregistered buffer") @@ -282,12 +297,14 @@ func (s *Server) HandlePut(key string, buf *Buffer, remoteStart uint64, size int n := C.cuobj_server_handle_put(s.csrv, ckey, buf.cbuf, C.uint64_t(remoteStart), C.size_t(size), cdescr, C.uint16_t(channel)) if n < 0 { - return 0, fmt.Errorf("rdma: handlePutObject failed (rc=%d)", n) + return 0, fmt.Errorf("rdma: handlePutObject failed (rc=%d): %w", n, syscall.Errno(-int64(n))) } return int64(n), nil } -// CloseSession tears down the RDMA session. +// CloseSession marks the session closed. The underlying library ties session +// teardown to the cuObjServer destructor, so the session is not actually torn +// down until Close is called. func (s *Server) CloseSession() { s.mu.Lock() defer s.mu.Unlock() diff --git a/rdma/rdma_stub.go b/rdma/rdma_stub.go index ce7337e4..545622cb 100644 --- a/rdma/rdma_stub.go +++ b/rdma/rdma_stub.go @@ -57,9 +57,6 @@ func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error) // StartSession always returns errNotSupported on this platform. func (s *Server) StartSession() error { return errNotSupported } -// InitRDMAConfig always returns errNotSupported on this platform. -func (s *Server) InitRDMAConfig(t RDMATunables) error { return errNotSupported } - // IsConnected always returns false on this platform. func (s *Server) IsConnected() bool { return false }