Merge pull request #2382 from versity/sis/cuobjserver-2.0.0-port

fix: port the cuObjServer wrapper to libcuobjserver 2.0.0

Makefile

@@ -74,7 +74,7 @@ $(BIN):
 	$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
 
 $(VGWRDMA_WRAPPER_LIB): cuwrapper/cuobjserver_wrapper.cpp cuwrapper/cuobjserver_wrapper.h
-	$(CXX) -c -fPIC \
+	$(CXX) -c -fPIC -std=c++17 \
 		-I$(CUOBJ_SERVER_INC_DIR) -Icuwrapper \
 		-o cuwrapper/cuobjserver_wrapper.o \
 		cuwrapper/cuobjserver_wrapper.cpp

cubackend/cuserver.go

@@ -59,10 +59,10 @@ func New(opts CuServerOpts, be backend.Backend) (*CuServer, error) {
 		return nil, fmt.Errorf("cuserver: rdma server: %w", err)
 	}
 
-	// StartSession is a no-op when the library manages session start
-	// internally (e.g. libcuobjserver v1.2.0 calls startRDMASession from
-	// the cuObjServer constructor). It is kept here for forward compatibility
-	// with library versions that require an explicit call.
+	// libcuobjserver starts the RDMA session inside the cuObjServer
+	// constructor, so this verifies the session came up rather than
+	// initiating it, and fails fast if the server constructed but did not
+	// connect.
 	if err := rdmaSrv.StartSession(); err != nil {
 		rdmaSrv.Close()
 		be.Shutdown()

cuwrapper/cuobjserver_wrapper.cpp

@@ -18,7 +18,6 @@
 #include "cuobjserver_wrapper.h"
 #include "cuobjserver.h"
 
-#include <dlfcn.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <atomic>
@@ -26,54 +25,24 @@
 #include <string>
 
 // ---------------------------------------------------------------------------
-// Session management — runtime symbol resolution
+// Session management
 //
-// libcuobjserver.so may export startRDMASession / closeRDMASession under one
-// of two C++ mangled names depending on when the library was compiled:
+// libcuobjserver does not expose an explicit RDMA session start/close entry
+// point. cuObjServer's constructor brings the session up and its destructor
+// tears it down.
 //
-//   Newer header layout (RDMAConnection base class):
-//     _ZN14RDMAConnection16startRDMASessionEv
-//     _ZN14RDMAConnection16closeRDMASessionEv
-//
-//   Older layout (method directly on cuObjServer):
-//     _ZN11cuObjServer16startRDMASessionEv
-//     _ZN11cuObjServer16closeRDMASessionEv
-//
-// We resolve lazily at first call so neither name needs to be present at
-// link time, and the code works with both library generations.
+// Earlier revisions of this wrapper resolved startRDMASession/closeRDMASession
+// at runtime via dlsym, trying both the RDMAConnection and cuObjServer mangled
+// names. Neither symbol was ever exported, so those lookups always failed and
+// the fallback paths below are what has always actually run. cuObjServer 2.0.0
+// deletes the RDMAConnection base class outright, so the lookup can no longer
+// succeed even in principle — it is gone rather than left as dead code.
 // ---------------------------------------------------------------------------
 
-typedef int  (*rdma_start_fn_t)(void *);
-typedef void (*rdma_close_fn_t)(void *);
-
 // Verbose wrapper logs are enabled when cuobj_server_set_telem_flags includes
 // info/debug bits (configured by cuserver -debug).
 static std::atomic<bool> g_verbose_logs{false};
 
-static rdma_start_fn_t find_start_rdma_session() {
-    void *proc = dlopen(nullptr, RTLD_LAZY);
-    if (!proc) return nullptr;
-    rdma_start_fn_t fn = reinterpret_cast<rdma_start_fn_t>(
-        dlsym(proc, "_ZN14RDMAConnection16startRDMASessionEv"));
-    if (!fn)
-        fn = reinterpret_cast<rdma_start_fn_t>(
-            dlsym(proc, "_ZN11cuObjServer16startRDMASessionEv"));
-    dlclose(proc);
-    return fn;
-}
-
-static rdma_close_fn_t find_close_rdma_session() {
-    void *proc = dlopen(nullptr, RTLD_LAZY);
-    if (!proc) return nullptr;
-    rdma_close_fn_t fn = reinterpret_cast<rdma_close_fn_t>(
-        dlsym(proc, "_ZN14RDMAConnection16closeRDMASessionEv"));
-    if (!fn)
-        fn = reinterpret_cast<rdma_close_fn_t>(
-            dlsym(proc, "_ZN11cuObjServer16closeRDMASessionEv"));
-    dlclose(proc);
-    return fn;
-}
-
 extern "C" {
 
 cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigned proto) {
@@ -85,8 +54,8 @@ cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigne
     }
 }
 
-// Build a cuObjRDMATunable from the flat C struct, shared by
-// cuobj_server_create_with_config and cuobj_server_init_rdma_config.
+// Build a cuObjRDMATunable from the flat C struct for
+// cuobj_server_create_with_config.
 static cuObjRDMATunable tunables_from_c(const cuobj_rdma_tunables_t *t) {
     cuObjRDMATunable config;
     config.setNumDcis(t->num_dcis);
@@ -122,27 +91,16 @@ void cuobj_server_destroy(cuobj_server_t *srv) {
 }
 
 int cuobj_server_start_session(cuobj_server_t *srv) {
-    static rdma_start_fn_t fn = find_start_rdma_session();
-    if (!fn) {
-        // Symbol not exported — library calls startRDMASession() internally
-        // from the cuObjServer constructor. Verify the session actually came
-        // up instead of unconditionally reporting success.
-        auto *s = reinterpret_cast<cuObjServer*>(srv);
-        return s->isConnected() ? 0 : -1;
-    }
-    int rc = fn(reinterpret_cast<void *>(srv));
-    if (rc != 0)
-        fprintf(stderr, "cuobjwrapper: startRDMASession returned %d\n", rc);
-    return rc;
+    // The cuObjServer constructor starts the session. Verify it actually came
+    // up instead of unconditionally reporting success.
+    auto *s = reinterpret_cast<cuObjServer*>(srv);
+    return s->isConnected() ? 0 : -1;
 }
 
 void cuobj_server_close_session(cuobj_server_t *srv) {
-    static rdma_close_fn_t fn = find_close_rdma_session();
-    if (!fn) {
-        // Symbol not exported — session cleanup handled by destructor.
-        return;
-    }
-    fn(reinterpret_cast<void *>(srv));
+    // Session teardown is owned by ~cuObjServer(); there is no separate close
+    // entry point. cuobj_server_destroy() is what actually closes the session.
+    (void)srv;
 }
 
 int cuobj_server_is_connected(cuobj_server_t *srv) {
@@ -231,17 +189,10 @@ void cuobj_server_shutdown_telemetry(void) {
 
 void cuobj_server_set_telem_flags(unsigned flags) {
     g_verbose_logs.store((flags & 0x0003u) != 0u);
-    cuObjServer::setTelemFlags(flags);
-}
-
-int cuobj_server_init_rdma_config(cuobj_server_t *srv, const cuobj_rdma_tunables_t *t) {
-    auto *s = reinterpret_cast<cuObjServer*>(srv);
-    try {
-        s->initRDMAConfigParams(tunables_from_c(t));
-        return 0;
-    } catch (...) {
-        return -1;
-    }
+    // 2.0.0 added a second mask selecting which operations get logged
+    // (CUOBJ_LOG_OP_GET / CUOBJ_LOG_OP_PUT). cuObjTelem initialises it to 0,
+    // so passing 0 keeps the 1.2.0 behaviour of no per-operation logging.
+    cuObjServer::setTelemFlags(flags, 0);
 }
 
 } // extern "C"

cuwrapper/cuobjserver_wrapper.h

@@ -39,6 +39,13 @@ cuobj_server_t* cuobj_server_create(const char *ip, unsigned short port, unsigne
 void            cuobj_server_destroy(cuobj_server_t *srv);
 
 // RDMA session
+//
+// libcuobjserver has never exported an explicit session start/close entry
+// point: the constructor brings the session up and the destructor tears it
+// down. cuObjServer 2.0.0 makes that official by deleting the RDMAConnection
+// base class that once declared start/closeRDMASession(). These two calls are
+// kept so callers can express lifecycle intent — start_session reports whether
+// the constructor-started session actually came up, close_session is a no-op.
 int  cuobj_server_start_session(cuobj_server_t *srv);
 void cuobj_server_close_session(cuobj_server_t *srv);
 int  cuobj_server_is_connected(cuobj_server_t *srv);
@@ -58,7 +65,9 @@ void     cuobj_server_free_channel(cuobj_server_t *srv, uint16_t channel_id);
 // Data transfer (synchronous, no poll_delay override)
 //
 // handleGetObject: RDMA WRITE server→client (serves a GET request)
-//   Returns bytes transferred or -1 on error.
+//   Returns bytes transferred, or a negative errno on failure
+//   (-EPROTO when the RDMA descriptor is malformed or its prefix does not
+//   match the server's protocol).
 ssize_t cuobj_server_handle_get(cuobj_server_t *srv,
                                 const char *key,
                                 cuobj_rdma_buffer_t *local_buf,
@@ -68,7 +77,9 @@ ssize_t cuobj_server_handle_get(cuobj_server_t *srv,
                                 uint16_t channel);
 
 // handlePutObject: RDMA READ client→server (serves a PUT request)
-//   Returns bytes transferred or -1 on error.
+//   Returns bytes transferred, or a negative errno on failure
+//   (-EPROTO when the RDMA descriptor is malformed or its prefix does not
+//   match the server's protocol).
 ssize_t cuobj_server_handle_put(cuobj_server_t *srv,
                                 const char *key,
                                 cuobj_rdma_buffer_t *local_buf,
@@ -78,12 +89,23 @@ ssize_t cuobj_server_handle_put(cuobj_server_t *srv,
                                 uint16_t channel);
 
 // Telemetry (optional)
+//
+// cuObjServer 2.0.0 split setTelemFlags into (log_flags, log_op_flags), where
+// the second mask (CUOBJ_LOG_OP_GET / CUOBJ_LOG_OP_PUT) enables per-operation
+// logging. That mask is not exposed here: cuObjTelem defaults it to 0, so
+// passing 0 reproduces the 1.2.0 behaviour this wrapper was written against.
 void cuobj_server_setup_telemetry(int use_otel);
 void cuobj_server_shutdown_telemetry(void);
 void cuobj_server_set_telem_flags(unsigned flags);
 
 // RDMA tunable parameters — flat C struct for CGO compatibility.
 // Field names and defaults match cuObjRDMATunableParam in cuobjrdma.h.
+//
+// cuObjServer 2.0.0 adds two further tunables, deliberately omitted here so
+// they keep their library defaults: `proto` (already defaults to
+// CUOBJ_PROTO_RDMA_DC_V1, the only protocol this gateway implements) and
+// `drain_wait_ms` (only consulted by the multi-VIP removeVip path, which this
+// wrapper does not expose).
 typedef struct {
     int           num_dcis;             // default 128
     unsigned      cq_depth;             // default 640
@@ -100,14 +122,11 @@ typedef struct {
     int           max_rd_atomic;        // default 0 (auto)
 } cuobj_rdma_tunables_t;
 
-// Apply RDMA tuning parameters to an existing connection object.
-// Takes effect on the next reconnection if called after session start.
-// Returns 0 on success, -1 on error.
-int cuobj_server_init_rdma_config(cuobj_server_t *srv, const cuobj_rdma_tunables_t *t);
-
 // Create a cuObjServer with tunable parameters applied before the session
-// starts. This is the preferred constructor when non-default tunables are
-// needed, since the library starts the RDMA session inside the constructor.
+// starts. This is the only way to set tunables: cuObjServer 2.0.0 deleted the
+// RDMAConnection base class, and with it initRDMAConfigParams(), so tunables
+// can no longer be applied to an already-constructed server. The library
+// starts the RDMA session inside the constructor regardless.
 cuobj_server_t* cuobj_server_create_with_config(const char *ip, unsigned short port, unsigned proto, const cuobj_rdma_tunables_t *t);
 
 #ifdef __cplusplus

rdma/rdma.go

@@ -19,7 +19,9 @@ package rdma
 
 /*
 #cgo CFLAGS: -I${SRCDIR}/../include -I${SRCDIR}/../cuwrapper
-#cgo LDFLAGS: -L${SRCDIR} -l:libcuobjwrapper.a -L/usr/lib64 -lcuobjserver -lstdc++ -ldl
+// Links against cuObjServer 2.x: the resulting binary records a
+// libcuobjserver.so.2 NEEDED entry and will not run against a 1.x install.
+#cgo LDFLAGS: -L${SRCDIR} -l:libcuobjwrapper.a -L/usr/lib64 -lcuobjserver -lstdc++
 #include "cuobjserver_wrapper.h"
 #include <stdlib.h>
 */
@@ -30,6 +32,7 @@ import (
 	"fmt"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"unsafe"
 )
 
@@ -112,6 +115,10 @@ func tunablesToC(t RDMATunables) C.cuobj_rdma_tunables_t {
 // Uses CUOBJ_PROTO_RDMA_DC_V1 (1001). If tunables is non-nil, the 4-argument
 // constructor is used so the tunable parameters apply to the initial session
 // started by the constructor. Pass nil to use library defaults.
+//
+// The constructor is the only point at which tunables can be applied:
+// libcuobjserver starts the RDMA session inside it, and cuObjServer 2.0.0
+// removed the after-the-fact initRDMAConfigParams() entry point.
 func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error) {
 	cip := C.CString(ip)
 	defer C.free(unsafe.Pointer(cip))
@@ -127,16 +134,21 @@ func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error)
 		return nil, fmt.Errorf("rdma: failed to create cuObjServer on %s:%d", ip, port)
 	}
 	srv := &Server{csrv: csrv}
-	// Some library versions start the session as part of construction;
-	// record that readiness so StartSession can be a no-op and Close/CloseSession
-	// use a consistent ownership model.
+	// The library starts the session as part of construction; record that
+	// readiness so StartSession can be a no-op and Close/CloseSession use a
+	// consistent ownership model.
 	if srv.IsConnected() {
 		srv.sessionOpen = true
 	}
 	return srv, nil
 }
 
-// StartSession initiates the RDMA listening session.
+// StartSession reports whether the RDMA session is up.
+//
+// libcuobjserver starts the session inside the cuObjServer constructor and has
+// never exposed an explicit start entry point, so this verifies the session
+// came up rather than initiating it. It is kept as an explicit lifecycle step
+// so callers can fail fast on a server that constructed but did not connect.
 // StartSession must not be called concurrently with CloseSession or Close.
 func (s *Server) StartSession() error {
 	s.mu.Lock()
@@ -153,7 +165,7 @@ func (s *Server) StartSession() error {
 			s.mu.Unlock()
 			return nil
 		}
-		return fmt.Errorf("rdma: startRDMASession failed (rc=%d)", rc)
+		return fmt.Errorf("rdma: RDMA session is not connected (rc=%d)", rc)
 	}
 	s.mu.Lock()
 	s.sessionOpen = true
@@ -161,15 +173,6 @@ func (s *Server) StartSession() error {
 	return nil
 }
 
-// InitRDMAConfig applies RDMA tuning parameters. Must be called before StartSession.
-func (s *Server) InitRDMAConfig(t RDMATunables) error {
-	ct := tunablesToC(t)
-	if rc := C.cuobj_server_init_rdma_config(s.csrv, &ct); rc != 0 {
-		return fmt.Errorf("rdma: initRDMAConfigParams failed (rc=%d)", rc)
-	}
-	return nil
-}
-
 // IsConnected returns the RDMA connection status.
 func (s *Server) IsConnected() bool {
 	return C.cuobj_server_is_connected(s.csrv) != 0
@@ -236,6 +239,12 @@ func (s *Server) FreeChannel(id uint16) {
 // HandleGet performs an RDMA WRITE (server→client) to serve a GET request.
 // The local buffer must already contain the data to send.
 // Returns bytes transferred.
+//
+// On failure the library returns a negative errno, which is wrapped into the
+// returned error: a malformed RDMA descriptor, or one whose prefix does not
+// match the server's protocol, reports syscall.EPROTO and can be matched with
+// errors.Is. That distinguishes a bad client-supplied descriptor from a
+// transport fault.
 func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int64, rdmaDescr string, channel uint16) (int64, error) {
 	if buf == nil || buf.cbuf == nil {
 		return 0, errors.New("rdma: invalid or deregistered buffer")
@@ -255,7 +264,7 @@ func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int
 	n := C.cuobj_server_handle_get(s.csrv, ckey, buf.cbuf,
 		C.uint64_t(remoteStart), C.size_t(size), cdescr, C.uint16_t(channel))
 	if n < 0 {
-		return 0, fmt.Errorf("rdma: handleGetObject failed (rc=%d)", n)
+		return 0, fmt.Errorf("rdma: handleGetObject failed (rc=%d): %w", n, syscall.Errno(-int64(n)))
 	}
 	return int64(n), nil
 }
@@ -263,6 +272,12 @@ func (s *Server) HandleGet(key string, buf *Buffer, remoteStart uint64, size int
 // HandlePut performs an RDMA READ (client→server) to serve a PUT request.
 // After return, the local buffer contains the data read from the client.
 // Returns bytes transferred.
+//
+// On failure the library returns a negative errno, which is wrapped into the
+// returned error: a malformed RDMA descriptor, or one whose prefix does not
+// match the server's protocol, reports syscall.EPROTO and can be matched with
+// errors.Is. That distinguishes a bad client-supplied descriptor from a
+// transport fault.
 func (s *Server) HandlePut(key string, buf *Buffer, remoteStart uint64, size int64, rdmaDescr string, channel uint16) (int64, error) {
 	if buf == nil || buf.cbuf == nil {
 		return 0, errors.New("rdma: invalid or deregistered buffer")
@@ -282,12 +297,14 @@ func (s *Server) HandlePut(key string, buf *Buffer, remoteStart uint64, size int
 	n := C.cuobj_server_handle_put(s.csrv, ckey, buf.cbuf,
 		C.uint64_t(remoteStart), C.size_t(size), cdescr, C.uint16_t(channel))
 	if n < 0 {
-		return 0, fmt.Errorf("rdma: handlePutObject failed (rc=%d)", n)
+		return 0, fmt.Errorf("rdma: handlePutObject failed (rc=%d): %w", n, syscall.Errno(-int64(n)))
 	}
 	return int64(n), nil
 }
 
-// CloseSession tears down the RDMA session.
+// CloseSession marks the session closed. The underlying library ties session
+// teardown to the cuObjServer destructor, so the session is not actually torn
+// down until Close is called.
 func (s *Server) CloseSession() {
 	s.mu.Lock()
 	defer s.mu.Unlock()

rdma/rdma_stub.go

@@ -57,9 +57,6 @@ func NewServer(ip string, port uint16, tunables *RDMATunables) (*Server, error)
 // StartSession always returns errNotSupported on this platform.
 func (s *Server) StartSession() error { return errNotSupported }
 
-// InitRDMAConfig always returns errNotSupported on this platform.
-func (s *Server) InitRDMAConfig(t RDMATunables) error { return errNotSupported }
-
 // IsConnected always returns false on this platform.
 func (s *Server) IsConnected() bool { return false }