versitygw/cuwrapper/rc/rc_ibv_host.cpp

/* Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
 *
 * SPDX-License-Identifier: MIT
 */

#include <cerrno>
#include "rc_ibv_host.h"

namespace hipObj {

IBVWrapper &IBVWrapper::instance() {
  static IBVWrapper w;
  return w;
}

IBVWrapper ibv __attribute__((init_priority(400)));

/* The inline verbs wrappers of verbs.h, reproduced: route each call
 * through the ops table of the context that owns the object. A NULL
 * slot means the provider does not implement the verb; report that
 * instead of jumping to address zero. */
static int dispatchPollCq(struct ibv_cq *cq, int num_entries,
                          struct ibv_wc *wc) {
  if (!cq || !cq->context || !cq->context->ops.poll_cq) {
    errno = ENOSYS;
    return -1;
  }
  return cq->context->ops.poll_cq(cq, num_entries, wc);
}

static int dispatchPostRecv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
                            struct ibv_recv_wr **bad_wr) {
  if (!qp || !qp->context || !qp->context->ops.post_recv) {
    if (bad_wr) *bad_wr = wr;
    return ENOSYS;
  }
  return qp->context->ops.post_recv(qp, wr, bad_wr);
}

static int dispatchPostSend(struct ibv_qp *qp, struct ibv_send_wr *wr,
                            struct ibv_send_wr **bad_wr) {
  if (!qp || !qp->context || !qp->context->ops.post_send) {
    if (bad_wr) *bad_wr = wr;
    return ENOSYS;
  }
  return qp->context->ops.post_send(qp, wr, bad_wr);
}

bool IBVWrapper::ensureLoaded() {
  std::lock_guard<std::mutex> guard(mtx_);
  if (loaded) {
    return true;
  }
  handle_ = dlopen("libibverbs.so.1", RTLD_NOW | RTLD_GLOBAL);
  if (handle_ == nullptr) {
    handle_ = dlopen("libibverbs.so", RTLD_NOW | RTLD_GLOBAL);
  }
  if (handle_ == nullptr) {
    return false;
  }
  auto load = [&](const char *name) -> void * {
    return dlsym(handle_, name);
  };
  funcs_.get_device_list =
      reinterpret_cast<struct ibv_device **(*)(int *)>(load("ibv_get_device_list"));
  funcs_.free_device_list =
      reinterpret_cast<void (*)(struct ibv_device **)>(load("ibv_free_device_list"));
  funcs_.open_device = reinterpret_cast<struct ibv_context *(*)(struct ibv_device *)>(
      load("ibv_open_device"));
  funcs_.close_device = reinterpret_cast<int (*)(struct ibv_context *)>(
      load("ibv_close_device"));
  funcs_.alloc_pd = reinterpret_cast<struct ibv_pd *(*)(struct ibv_context *)>(
      load("ibv_alloc_pd"));
  funcs_.dealloc_pd = reinterpret_cast<int (*)(struct ibv_pd *)>(
      load("ibv_dealloc_pd"));
  funcs_.reg_mr = reinterpret_cast<struct ibv_mr *(*)(struct ibv_pd *, void *,
                                                      size_t, int)>(load("ibv_reg_mr"));
  /* Host-only build: both spellings land on the plain verbs call. */
  funcs_.reg_mr_host = funcs_.reg_mr;
  funcs_.dereg_mr = reinterpret_cast<int (*)(struct ibv_mr *)>(
      load("ibv_dereg_mr"));
  funcs_.create_cq = reinterpret_cast<struct ibv_cq *(*)(
      struct ibv_context *, int, void *, struct ibv_comp_channel *, int)>(
      load("ibv_create_cq"));
  funcs_.destroy_cq = reinterpret_cast<int (*)(struct ibv_cq *)>(
      load("ibv_destroy_cq"));
  funcs_.create_qp = reinterpret_cast<struct ibv_qp *(*)(
      struct ibv_pd *, struct ibv_qp_init_attr *)>(load("ibv_create_qp"));
  funcs_.destroy_qp = reinterpret_cast<int (*)(struct ibv_qp *)>(
      load("ibv_destroy_qp"));
  funcs_.modify_qp = reinterpret_cast<int (*)(struct ibv_qp *,
                                              struct ibv_qp_attr *, int)>(
      load("ibv_modify_qp"));
  funcs_.poll_cq = reinterpret_cast<int (*)(struct ibv_cq *, int,
                                            struct ibv_wc *)>(load("ibv_poll_cq"));
  funcs_.query_device = reinterpret_cast<int (*)(
      struct ibv_context *, struct ibv_device_attr *)>(load("ibv_query_device"));
  funcs_.query_port = reinterpret_cast<int (*)(
      struct ibv_context *, uint8_t, struct ibv_port_attr *)>(
      load("ibv_query_port"));
  funcs_.query_gid = reinterpret_cast<int (*)(
      struct ibv_context *, uint8_t, int, union ibv_gid *)>(
      load("ibv_query_gid"));
  funcs_.post_recv = reinterpret_cast<int (*)(struct ibv_qp *,
      struct ibv_recv_wr *, struct ibv_recv_wr **)>(load("ibv_post_recv"));
  funcs_.post_send = reinterpret_cast<int (*)(struct ibv_qp *,
      struct ibv_send_wr *, struct ibv_send_wr **)>(load("ibv_post_send"));
  loaded = funcs_.get_device_list != nullptr && funcs_.open_device != nullptr &&
           funcs_.alloc_pd != nullptr && funcs_.create_qp != nullptr &&
           funcs_.modify_qp != nullptr;
  if (loaded) {
    /* poll_cq/post_send/post_recv are static inline wrappers in
     * modern verbs.h that dispatch through the object's own
     * context->ops table, so dlsym cannot find them on rdma-core
     * 61+. Dispatch the same way, per call: the provider behind a
     * QP or CQ is whichever opened its context, and a host with
     * more than one HCA has more than one provider. */
    funcs_.poll_cq = dispatchPollCq;
    funcs_.post_recv = dispatchPostRecv;
    funcs_.post_send = dispatchPostSend;
  }
  if (loaded) {
    /* Mirror into the member seam for direct ibv.x() calls. */
    get_device_list = funcs_.get_device_list;
    free_device_list = funcs_.free_device_list;
    open_device = funcs_.open_device;
    close_device = funcs_.close_device;
    alloc_pd = funcs_.alloc_pd;
    dealloc_pd = funcs_.dealloc_pd;
    reg_mr = funcs_.reg_mr;
    reg_mr_host = funcs_.reg_mr_host;
    dereg_mr = funcs_.dereg_mr;
    create_cq = funcs_.create_cq;
    destroy_cq = funcs_.destroy_cq;
    create_qp = funcs_.create_qp;
    destroy_qp = funcs_.destroy_qp;
    modify_qp = funcs_.modify_qp;
    poll_cq = funcs_.poll_cq;
    query_device = funcs_.query_device;
    query_port = funcs_.query_port;
    query_gid = funcs_.query_gid;
    post_recv = funcs_.post_recv;
    post_send = funcs_.post_send;
  }
  return loaded;
}

}  // namespace hipObj