rdma: port the hipObject v2 RC session core

Port the hipObject v2 reliable-connection session core into
cuwrapper/rc: the session table and state machine, the wire
codec for the hipobj-rc-v2 headers, request parsing, the
injectable clock and randomness sources, the transport layer
(QP/CQ lifecycle, RTR/RTS transitions, staging registration),
the data phase (RDMA write with immediate for GET, receive with
immediate for PUT), the RDMA token codec, and the dynamically
loaded ibverbs shim (ibv-core.h plus the dlopen host binding).

The sources are a port of the upstream hipObject v2 core, kept
close to the original so the two trees can be diffed during
review. Nothing links against them yet; a Makefile rule builds
the objects into rdma/librcserver.a for the ABI layer that
follows.

Signed-off-by: Jihyeon Gim <potatogim@potatogim.net>

cuwrapper/rc/v2_data_phase.cpp

@@ -0,0 +1,228 @@
+/* Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) Gluesys Inc. and Jihyeon Gim. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "v2_data_phase.h"
+
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+
+#include <arpa/inet.h>
+
+#include "rc_ibv_host.h"
+
+namespace hipObj {
+namespace v2 {
+
+namespace {
+
+constexpr int kAccess = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE |
+                        IBV_ACCESS_REMOTE_READ;
+
+/* Completion markers posted with every work request. */
+constexpr uint64_t kWrRecv = 0x5245435632494d4dULL;  /* RECV2IMM */
+constexpr uint64_t kWrWrite = 0x57524954454d4d47ULL; /* WRITEMM */
+
+uint64_t clockNowMs() {
+  struct timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+  return static_cast<uint64_t>(ts.tv_sec) * 1000 +
+         static_cast<uint64_t>(ts.tv_nsec) / 1000000;
+}
+
+} // namespace
+
+bool stagePutBuffer(V2Session& s, size_t size, struct ibv_pd* pd) {
+  if (s.staging != nullptr) {
+    return s.stagingMr != nullptr || pd == nullptr;
+  }
+  void* buf = std::malloc(size ? size : 1);
+  if (buf == nullptr) {
+    return false;
+  }
+  /* Without a PD (transport-free host) the buffer stages without
+   * an MR; the data phase is a no-op there anyway.
+   *
+   * Prefer the device-registered path: providers register the
+   * buffer with the device (dmabuf on GPU hosts, plain
+   * ibv_reg_mr otherwise) so peers can reach it via rkey. The
+   * host-only registration is a fallback for wrappers whose
+   * device path needs an unavailable GPU runtime. */
+  struct ibv_mr* mr = nullptr;
+  if (pd != nullptr) {
+    mr = ibv.reg_mr(pd, buf, size, kAccess);
+    if (mr == nullptr) {
+      mr = ibv.reg_mr_host(pd, buf, size, kAccess);
+    }
+    if (mr == nullptr) {
+      std::free(buf);
+      return false;
+    }
+  }
+  s.staging = buf;
+  s.stagingMr = mr;
+  return true;
+}
+
+void releaseStaging(V2Session& s) {
+  /* The caller must have quiesced or destroyed the session QP
+   * first: a posted work request can still reference the MR
+   * until the QP is gone. dereg failures leave the MR leaked
+   * (and logged) rather than freeing memory the NIC may touch. */
+  if (s.stagingMr != nullptr) {
+    if (ibv.dereg_mr(s.stagingMr) != 0) {
+      fprintf(stderr, "v2: staging dereg failed; leaking buffer\n");
+      s.staging = nullptr; /* MR is dead to us either way */
+    }
+    s.stagingMr = nullptr;
+  }
+  if (s.staging != nullptr) {
+    std::free(s.staging);
+    s.staging = nullptr;
+  }
+}
+
+bool postRecvForImm(struct ibv_qp* qp, struct ibv_mr* mr, size_t len) {
+  struct ibv_sge sge;
+  std::memset(&sge, 0, sizeof(sge));
+  sge.addr = reinterpret_cast<uintptr_t>(mr->addr);
+  sge.length = static_cast<uint32_t>(len);
+  sge.lkey = mr->lkey;
+
+  struct ibv_recv_wr wr;
+  std::memset(&wr, 0, sizeof(wr));
+  wr.wr_id = kWrRecv;
+  wr.sg_list = &sge;
+  wr.num_sge = 1;
+
+  struct ibv_recv_wr* bad = nullptr;
+  return ibv.post_recv(qp, &wr, &bad) == 0;
+}
+
+bool postWriteWithImm(struct ibv_qp* qp, struct ibv_mr* src,
+                      uint64_t remoteAddr, uint32_t rkey, size_t len,
+                      uint32_t immData) {
+  /* GET delivery: server pushes the object into the client MR
+   * with the session cookie as the immediate. */
+  struct ibv_sge sge;
+  std::memset(&sge, 0, sizeof(sge));
+  sge.addr = reinterpret_cast<uintptr_t>(src->addr);
+  sge.length = static_cast<uint32_t>(len);
+  sge.lkey = src->lkey;
+
+  struct ibv_send_wr wr;
+  std::memset(&wr, 0, sizeof(wr));
+  wr.wr_id = kWrWrite;
+  wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
+  wr.send_flags = IBV_SEND_SIGNALED;
+  wr.imm_data = htonl(immData);
+  wr.wr.rdma.remote_addr = remoteAddr;
+  wr.wr.rdma.rkey = rkey;
+  wr.sg_list = &sge;
+  wr.num_sge = 1;
+
+  struct ibv_send_wr* bad = nullptr;
+  return ibv.post_send(qp, &wr, &bad) == 0;
+}
+
+/* Polls the CQ for one completion matching `expectWr`, bounded by
+ * an absolute deadline on the monotonic clock. */
+enum class PollOutcome { Ok, Timeout, Error, Mismatch };
+PollOutcome pollCqUntil(struct ibv_cq* cq, uint64_t deadlineMs,
+                        uint64_t expectWr, struct ibv_wc* out) {
+  for (;;) {
+    int n = ibv.poll_cq(cq, 1, out);
+    if (n > 0) {
+      /* Providers may rewrite the wr_id on emulated paths; the
+       * opcode + immediate + length identify the completion. */
+      (void)expectWr;
+      return PollOutcome::Ok;
+    }
+    if (n < 0) {
+      return PollOutcome::Error;
+    }
+    if (clockNowMs() >= deadlineMs) {
+      return PollOutcome::Timeout;
+    }
+    struct timespec ts = {0, 2 * 1000 * 1000};
+    nanosleep(&ts, nullptr);
+  }
+}
+
+DataPhaseResult runDataPhase(V2Session& s, uint64_t deadlineMs,
+                             DataPhaseStats& stats) {
+  const bool noTransport = s.qp == nullptr && s.cq == nullptr;
+  if (noTransport || s.clientQpn == 0) {
+    /* Control-plane-only session (unit tests, reference
+     * harness): both objects absent or the client advertised no
+     * QP. A half-wired session is not accepted here. */
+    stats.bytes = s.size;
+    stats.cookie = s.cookie;
+    return DataPhaseResult::Ok;
+  }
+  if (s.qp == nullptr || s.cq == nullptr || s.stagingMr == nullptr) {
+    return DataPhaseResult::WireFail;
+  }
+
+  struct ibv_wc wc;
+  PollOutcome po;
+
+  if (s.op == "PUT") {
+    /* The client writes into the server staging MR and signals
+     * the session cookie. The server is the responder here, so
+     * requester-side retry exhaustion never surfaces in this CQ;
+     * any completion error or mismatch is a wire defect. */
+    if (!postRecvForImm(s.qp, s.stagingMr, static_cast<size_t>(s.size))) {
+      return DataPhaseResult::WireFail;
+    }
+    po = pollCqUntil(s.cq, deadlineMs, kWrRecv, &wc);
+    if (po == PollOutcome::Timeout) {
+      return DataPhaseResult::Timeout;
+    }
+    if (po != PollOutcome::Ok || wc.status != IBV_WC_SUCCESS ||
+        wc.opcode != IBV_WC_RECV_RDMA_WITH_IMM ||
+        (wc.wc_flags & IBV_WC_WITH_IMM) == 0 ||
+        ntohl(wc.imm_data) != s.cookie || wc.byte_len != s.size) {
+      return DataPhaseResult::VerifyFail;
+    }
+    stats.bytes = wc.byte_len;
+    stats.cookie = s.cookie;
+    return DataPhaseResult::Ok;
+  }
+
+  /* GET: push the staged object to the client MR with the
+   * cookie as the immediate; the client's receive consumes it. */
+  if (s.clientMrAddr == 0 || s.clientMrRkey == 0) {
+    return DataPhaseResult::WireFail;
+  }
+  if (!postWriteWithImm(s.qp, s.stagingMr, s.clientMrAddr, s.clientMrRkey,
+                        static_cast<size_t>(s.size), s.cookie)) {
+    return DataPhaseResult::WireFail;
+  }
+  po = pollCqUntil(s.cq, deadlineMs, kWrWrite, &wc);
+  if (po == PollOutcome::Timeout) {
+    return DataPhaseResult::Timeout;
+  }
+  if (po == PollOutcome::Ok &&
+      (wc.status == IBV_WC_RNR_RETRY_EXC_ERR ||
+       wc.status == IBV_WC_RETRY_EXC_ERR)) {
+    /* The server is the requester for the RDMA write: these mean
+     * the peer's receive queue was not armed or the peer did not
+     * answer, which is retryable from a fresh pairing rather
+     * than a wire defect. */
+    return DataPhaseResult::Busy;
+  }
+  if (po != PollOutcome::Ok || wc.status != IBV_WC_SUCCESS ||
+      wc.opcode != IBV_WC_RDMA_WRITE) {
+    return DataPhaseResult::VerifyFail;
+  }
+  stats.bytes = s.size;
+  stats.cookie = s.cookie;
+  return DataPhaseResult::Ok;
+}
+
+} // namespace v2
+} // namespace hipObj