diff --git a/seaweedfs-rdma-sidecar/rdma-engine/Cargo.lock b/seaweedfs-rdma-sidecar/rdma-engine/Cargo.lock index 6c2b85197..12da28885 100644 --- a/seaweedfs-rdma-sidecar/rdma-engine/Cargo.lock +++ b/seaweedfs-rdma-sidecar/rdma-engine/Cargo.lock @@ -161,6 +161,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.2", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -230,6 +253,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.1" @@ -248,7 +280,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -278,6 +310,17 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.45" @@ -530,6 +573,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "half" version = "2.6.0" @@ -561,6 +610,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "iana-time-zone" version = "0.1.63" @@ -655,6 +713,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.175" @@ -677,6 +741,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -723,6 +793,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -910,6 +989,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plotters" version = "0.3.7" @@ -947,6 +1032,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.100" @@ -1073,6 +1168,7 @@ dependencies = [ "nix", "parking_lot", "proptest", + "rdma-sys", "rmp-serde", "serde", "tempfile", @@ -1084,6 +1180,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "rdma-sys" +version = "0.3.0" +dependencies = [ + "bindgen", + "libc", + "memoffset", + "paste", + "pkg-config", +] + [[package]] name = "redox_syscall" version = "0.5.17" @@ -1171,6 +1278,25 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.9.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.0.8" @@ -1180,7 +1306,7 @@ dependencies = [ "bitflags 2.9.2", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.9.4", "windows-sys 0.60.2", ] @@ -1338,7 +1464,7 @@ dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix", + "rustix 1.0.8", "windows-sys 0.59.0", ] @@ -1671,6 +1797,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -1688,7 +1826,7 @@ checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", - "windows-link", + "windows-link 0.1.3", "windows-result", "windows-strings", ] @@ -1721,13 +1859,19 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-result" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -1736,7 +1880,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -1766,6 +1910,15 @@ dependencies = [ "windows-targets 0.53.3", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1788,7 +1941,7 @@ version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ - "windows-link", + "windows-link 0.1.3", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", diff --git a/seaweedfs-rdma-sidecar/rdma-engine/Cargo.toml b/seaweedfs-rdma-sidecar/rdma-engine/Cargo.toml index 5b355fffc..98f38f56f 100644 --- a/seaweedfs-rdma-sidecar/rdma-engine/Cargo.toml +++ b/seaweedfs-rdma-sidecar/rdma-engine/Cargo.toml @@ -10,6 +10,13 @@ license = "Apache-2.0" name = "rdma-engine-server" path = "src/main.rs" +# Phase-1 proof: a real IBV_WR_RDMA_READ over SoftRoCE loopback (requires Linux + +# libibverbs/librdmacm). Build with: cargo build --features real-rdma --bin rdma-loopback +[[bin]] +name = "rdma-loopback" +path = "src/bin/rdma_loopback.rs" +required-features = ["real-rdma"] + [lib] name = "rdma_engine" path = "src/lib.rs" @@ -48,6 +55,10 @@ parking_lot = "0.12" # Fast mutexes nix = { version = "0.27", features = ["mman"] } # Unix domain sockets and system calls async-trait = "0.1" # Async traits +# Real RDMA via libibverbs + librdmacm (raw FFI). Optional, Linux-only; pulled in +# by the `real-rdma` feature. Used by src/rdma_real.rs (RealRdmaContext). +rdma-sys = { version = "0.3", optional = true } + # Configuration clap = { version = "4.0", features = ["derive"] } config = "0.13" @@ -61,6 +72,7 @@ tempfile = "3.0" default = ["mock-ucx"] mock-ucx = [] real-ucx = [] # UCX integration for production RDMA +real-rdma = ["dep:rdma-sys"] # real libibverbs/rdma_cm datapath (SoftRoCE/real NIC) [profile.release] opt-level = 3 @@ -72,3 +84,6 @@ panic = "abort" [package.metadata.docs.rs] features = ["real-rdma"] + +[patch.crates-io] +rdma-sys = { path = "vendor/rdma-sys" } diff --git a/seaweedfs-rdma-sidecar/rdma-engine/REAL_RDMA.md b/seaweedfs-rdma-sidecar/rdma-engine/REAL_RDMA.md new file mode 100644 index 000000000..11c6fcbe0 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/REAL_RDMA.md @@ -0,0 +1,58 @@ +# Real RDMA datapath (`real-rdma` feature) — proof of plumbing + +The engine's default backend is a **mock** (`src/rdma.rs` `MockRdmaContext`): its +`post_read` fabricates `byte = i % 256` and never touches a wire (the Go sidecar +in `../pkg/rdma/client.go` even discards the engine's buffer and re-fakes the +same pattern). This module replaces that with a **real** libibverbs / `rdma_cm` +datapath, validated over SoftRoCE (rxe). + +## What's here + +- `src/rdma_real.rs` + - `RealRdmaContext` — the real client backend: `connect` (via `rdma_cm` / + `rdma_create_ep`), `register_memory`, `post_read` (a genuine + `IBV_WR_RDMA_READ` via `rdma_post_read`), `poll_completion`, `device_info`. + - `RealResponder` — a throwaway loopback target: registers a buffer for + `REMOTE_READ` (`rdma_reg_read`) and hands the client its `{addr, rkey, len}`. +- `src/bin/rdma_loopback.rs` — the proof: a responder registers a buffer with a + verifiable pattern (a `REALRDMA` magic header — deliberately **not** the mock's + `i%256`), the client RDMA-READs it, and asserts the bytes match. + +Connection setup and QP state transitions use `rdma_cm` (much simpler than a +manual GID/QPN/LID handshake for RoCEv2 addressing). + +## Build & run (Linux + SoftRoCE only) + +Requires `libibverbs-dev librdmacm-dev ibverbs-providers ibverbs-utils +rdma-core clang libclang-dev`, and a SoftRoCE link: + +```sh +sudo modprobe rdma_rxe rdma_ucm +sudo rdma link add rxe0 type rxe netdev eth0 # eth0 = your primary netdev +ibv_rc_pingpong -d rxe0 -g 1 # sanity-check the fabric + +cargo build --features real-rdma --bin rdma-loopback +sudo ./target/debug/rdma-loopback 7471 1048576 # rxe_ip = eth0's IP +# => REAL-RDMA-READ-SUCCESS, dest head "REALRDMA", bytes match +``` + +(rxe on a low-vCPU VM is a **correctness** proof, not a representative benchmark.) + +## Why `vendor/rdma-sys` + +`rdma-sys 0.3.0` (crates.io) pins **bindgen 0.59.2**, which cannot parse the +`rdma-core 56` headers (`ib_uverbs_flow_action_esp_encap_union_… is not a valid +Ident`). `vendor/rdma-sys` is rdma-sys 0.3.0 with bindgen bumped to 0.69.4 (and +the two methods removed in modern bindgen — `rustfmt_bindings` / `size_t_is_usize` +— dropped from its `build.rs`); `Cargo.toml` redirects to it via +`[patch.crates-io]`. This is a temporary workaround — upstreaming a bindgen bump +to rdma-sys, or moving to a maintained bindings crate, is the cleaner long-term +fix. + +## Not done (the real RDMA *win* is multi-week) + +This is Phase-1 proof-of-plumbing only. A demonstrable zero-copy read of real +volume data still needs: a needle-backed RDMA **target** in the volume server +(none exists today), a control-plane handshake to publish `{remote_addr, rkey}`, +a breaking IPC-contract change (Rust + Go, in lockstep), mount wiring, and then +the zero-copy READ path. See the scoping notes for the phased plan. diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/bin/rdma_loopback.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/bin/rdma_loopback.rs new file mode 100644 index 000000000..08e88fe94 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/src/bin/rdma_loopback.rs @@ -0,0 +1,95 @@ +//! Phase-1 proof: a real `IBV_WR_RDMA_READ` over SoftRoCE (rxe) loopback. +//! +//! Spawns a [`RealResponder`] that registers a buffer holding a verifiable +//! pattern (with a `REALRDMA` magic header — deliberately NOT the mock's `i%256`), +//! then a [`RealRdmaContext`] client connects, RDMA-READs the buffer, and asserts +//! the bytes match. Proves the engine moves real bytes over a genuine RDMA verb. +//! +//! rmmod-free run (after SoftRoCE is up): +//! cargo run --features real-rdma --bin rdma-loopback -- [port] [len] +//! +//! `` must be an IP on the netdev backing the rxe link (e.g. eth0's IP), +//! not 127.0.0.1 (rxe is bound to that netdev, not loopback). + +use std::sync::mpsc; +use std::time::Instant; + +use rdma_engine::rdma_real::{RealRdmaContext, RealResponder}; + +fn main() { + let args: Vec = std::env::args().collect(); + let ip = args.get(1).cloned().unwrap_or_else(|| "127.0.0.1".into()); + let port: u16 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(7471); + let len: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1 << 20); + + // A verifiable pattern, distinct from the mock's i%256, with a magic header. + let mut src = vec![0u8; len]; + for (i, b) in src.iter_mut().enumerate() { + *b = (i.wrapping_mul(131).wrapping_add(17) & 0xff) as u8; + } + let magic = b"REALRDMA"; + src[..magic.len()].copy_from_slice(magic); + let expected = src.clone(); + + let (ready_tx, ready_rx) = mpsc::channel::<()>(); + let sip = ip.clone(); + let server = std::thread::spawn(move || { + let mut data = src; + if let Err(e) = RealResponder::serve_once(&sip, port, &mut data, move || { + let _ = ready_tx.send(()); + }) { + eprintln!("responder error: {e}"); + } + }); + // Wait until the responder is listening before connecting. + let _ = ready_rx.recv(); + + let t0 = Instant::now(); + let (ctx, raddr, rkey, rlen) = match RealRdmaContext::connect(&ip, port) { + Ok(v) => v, + Err(e) => { + eprintln!("client connect failed: {e}"); + let _ = server.join(); + std::process::exit(1); + } + }; + println!( + "connected: device={:?} remote MR addr=0x{raddr:x} rkey=0x{rkey:x} len={rlen}", + ctx.device_info().name + ); + + let mut dest = vec![0u8; rlen]; + let (_mrinfo, mr) = ctx.register_memory(&mut dest).expect("register dest"); + ctx.post_read(&mut dest, mr, raddr, rkey, 1).expect("post_read"); + let wc = ctx.poll_completion().expect("poll completion"); + let dt = t0.elapsed(); + ctx.finish(); + let _ = server.join(); + + let head: String = dest[..16.min(dest.len())] + .iter() + .map(|b| format!("{b:02x}")) + .collect::>() + .join(" "); + println!( + "RDMA READ done: status={:?} byte_len={} dest head: {head} (\"{}\")", + wc.status, + wc.byte_len, + String::from_utf8_lossy(&dest[..magic.len().min(dest.len())]) + ); + + if dest == expected { + let mibps = (rlen as f64 / (1024.0 * 1024.0)) / dt.as_secs_f64(); + println!("VERIFY OK: {rlen} bytes match the responder's registered buffer"); + println!("({mibps:.1} MiB/s incl. connection setup; rxe on a 2-vCPU VM is a correctness proof, not a benchmark)"); + println!("REAL-RDMA-READ-SUCCESS"); + } else { + let mismatch = dest + .iter() + .zip(expected.iter()) + .position(|(a, b)| a != b) + .unwrap_or(0); + eprintln!("VERIFY FAILED: first mismatch at byte {mismatch}"); + std::process::exit(1); + } +} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/lib.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/lib.rs index c92dcf91a..f80180773 100644 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/lib.rs +++ b/seaweedfs-rdma-sidecar/rdma-engine/src/lib.rs @@ -27,6 +27,8 @@ use anyhow::Result; pub mod ucx; pub mod rdma; +#[cfg(feature = "real-rdma")] +pub mod rdma_real; // real libibverbs/rdma_cm RDMA datapath (SoftRoCE/real NIC) pub mod ipc; pub mod session; pub mod memory; diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/rdma_real.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/rdma_real.rs new file mode 100644 index 000000000..7ff53d074 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/src/rdma_real.rs @@ -0,0 +1,360 @@ +//! Real RDMA datapath over libibverbs + librdmacm (`rdma_cm`). +//! +//! Phase-1 proof-of-plumbing: a genuine `IBV_WR_RDMA_READ` (via the librdmacm +//! convenience verb `rdma_post_read`) that pulls *real* bytes out of a remote +//! peer's registered memory region over SoftRoCE (rxe) — replacing the mock +//! engine that fabricated `byte = i % 256` and never touched a wire. +//! +//! [`RealRdmaContext`] is the client (the engine's real backend: connect / +//! register_memory / post_read / poll_completion / device_info); [`RealResponder`] +//! is a throwaway loopback target that registers a buffer for `REMOTE_READ` and +//! hands the client its `{addr, rkey, len}`. Connection setup and QP state +//! transitions are handled by `rdma_cm` (much simpler than a manual GID/QPN/LID +//! handshake for RoCEv2). Linux + libibverbs/librdmacm only (`real-rdma` feature). + +#![allow(clippy::missing_safety_doc)] + +use std::ffi::{CStr, CString}; +use std::os::raw::{c_int, c_void}; +use std::ptr::null_mut; + +use rdma_sys::*; + +use crate::error::{RdmaError, RdmaResult}; +use crate::rdma::{CompletionStatus, MemoryRegion, RdmaDeviceInfo, RdmaOp, WorkCompletion}; + +/// Metadata the responder hands the client so it can RDMA-READ the source MR. +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct RemoteInfo { + addr: u64, + rkey: u32, + len: u32, +} + +const META_LEN: usize = std::mem::size_of::(); + +fn op_failed(op: &str) -> RdmaError { + RdmaError::OperationFailed { + operation: format!("{op}: {}", std::io::Error::last_os_error()), + status: -1, + } +} + +unsafe fn qp_init_attr() -> ibv_qp_init_attr { + let mut a: ibv_qp_init_attr = std::mem::zeroed(); + a.cap.max_send_wr = 8; + a.cap.max_recv_wr = 8; + a.cap.max_send_sge = 1; + a.cap.max_recv_sge = 1; + a.sq_sig_all = 1; + a +} + +// Allow at least one outstanding RDMA READ in each direction. +unsafe fn conn_param() -> rdma_conn_param { + let mut cp: rdma_conn_param = std::mem::zeroed(); + cp.responder_resources = 1; + cp.initiator_depth = 1; + cp.retry_count = 7; + cp.rnr_retry_count = 7; + cp +} + +unsafe fn query_device(id: *mut rdma_cm_id) -> RdmaDeviceInfo { + let ctx = (*id).verbs; + let mut attr: ibv_device_attr = std::mem::zeroed(); + let mut name = String::from("unknown"); + if !ctx.is_null() { + ibv_query_device(ctx, &mut attr); + let dev = (*ctx).device; + if !dev.is_null() { + name = CStr::from_ptr((*dev).name.as_ptr()) + .to_string_lossy() + .into_owned(); + } + } + RdmaDeviceInfo { + name, + vendor_id: attr.vendor_id, + vendor_part_id: attr.vendor_part_id, + hw_ver: attr.hw_ver, + max_mr: attr.max_mr as u32, + max_qp: attr.max_qp as u32, + max_cq: attr.max_cq as u32, + max_mr_size: attr.max_mr_size, + port_gid: String::new(), + port_lid: 0, + } +} + +/// Busy-poll a librdmacm completion helper (`rdma_get_send_comp` / +/// `rdma_get_recv_comp`) until it returns a completion or an error. +unsafe fn wait_comp( + f: unsafe fn(*mut rdma_cm_id, *mut ibv_wc) -> c_int, + id: *mut rdma_cm_id, + what: &str, +) -> RdmaResult { + let mut wc: ibv_wc = std::mem::zeroed(); + loop { + let n = f(id, &mut wc); + if n < 0 { + return Err(op_failed(what)); + } + if n > 0 { + break; + } + } + if wc.status != ibv_wc_status::IBV_WC_SUCCESS { + return Err(RdmaError::OperationFailed { + operation: format!("{what}: wc status {}", wc.status), + status: wc.status as i32, + }); + } + Ok(wc) +} + +/// The real RDMA client backend. +pub struct RealRdmaContext { + id: *mut rdma_cm_id, + device: RdmaDeviceInfo, +} + +impl RealRdmaContext { + /// Connect to a responder at `ip:port` and receive the `{addr, rkey, len}` + /// of its RDMA-READable source memory region. + pub fn connect(ip: &str, port: u16) -> RdmaResult<(Self, u64, u32, usize)> { + unsafe { + let node = CString::new(ip).map_err(|_| op_failed("bad ip"))?; + let svc = CString::new(port.to_string()).unwrap(); + let mut hints: rdma_addrinfo = std::mem::zeroed(); + hints.ai_port_space = rdma_port_space::RDMA_PS_TCP as c_int; + let mut res: *mut rdma_addrinfo = null_mut(); + if rdma_getaddrinfo(node.as_ptr(), svc.as_ptr(), &hints, &mut res) != 0 { + return Err(op_failed("rdma_getaddrinfo(client)")); + } + let mut attr = qp_init_attr(); + let mut id: *mut rdma_cm_id = null_mut(); + let r = rdma_create_ep(&mut id, res, null_mut(), &mut attr); + rdma_freeaddrinfo(res); + if r != 0 { + return Err(op_failed("rdma_create_ep(client)")); + } + + // Post a receive for the 16-byte RemoteInfo before connecting. + let mut meta = RemoteInfo::default(); + let meta_ptr = (&mut meta as *mut RemoteInfo).cast::(); + let meta_mr = rdma_reg_msgs(id, meta_ptr, META_LEN); + if meta_mr.is_null() { + rdma_destroy_ep(id); + return Err(op_failed("rdma_reg_msgs(meta)")); + } + if rdma_post_recv(id, null_mut(), meta_ptr, META_LEN, meta_mr) != 0 { + rdma_destroy_ep(id); + return Err(op_failed("rdma_post_recv(meta)")); + } + + let mut cp = conn_param(); + if rdma_connect(id, &mut cp) != 0 { + rdma_destroy_ep(id); + return Err(op_failed("rdma_connect")); + } + wait_comp(rdma_get_recv_comp, id, "recv(meta)")?; + rdma_dereg_mr(meta_mr); + + let device = query_device(id); + Ok(( + RealRdmaContext { id, device }, + meta.addr, + meta.rkey, + meta.len as usize, + )) + } + } + + /// Register a local buffer (LOCAL_WRITE) as the destination of an RDMA READ. + /// Returns the engine's `MemoryRegion` plus the raw `ibv_mr` needed to post. + pub fn register_memory(&self, buf: &mut [u8]) -> RdmaResult<(MemoryRegion, *mut ibv_mr)> { + unsafe { + let mr = rdma_reg_msgs(self.id, buf.as_mut_ptr().cast(), buf.len()); + if mr.is_null() { + return Err(op_failed("rdma_reg_msgs(dest)")); + } + Ok(( + MemoryRegion { + addr: buf.as_ptr() as u64, + rkey: (*mr).rkey, + lkey: (*mr).lkey, + size: buf.len(), + registered: true, + }, + mr, + )) + } + } + + /// Post a genuine `IBV_WR_RDMA_READ`: pull `buf.len()` bytes from the remote + /// `{remote_addr, rkey}` into the local buffer described by `mr`. + pub fn post_read( + &self, + buf: &mut [u8], + mr: *mut ibv_mr, + remote_addr: u64, + rkey: u32, + wr_id: u64, + ) -> RdmaResult<()> { + unsafe { + let flags = ibv_send_flags::IBV_SEND_SIGNALED.0 as c_int; + if rdma_post_read( + self.id, + wr_id as *mut c_void, + buf.as_mut_ptr().cast(), + buf.len(), + mr, + flags, + remote_addr, + rkey, + ) != 0 + { + return Err(op_failed("rdma_post_read")); + } + Ok(()) + } + } + + /// Wait for the RDMA READ completion (it lands on the send queue). + pub fn poll_completion(&self) -> RdmaResult { + unsafe { + let wc = wait_comp(rdma_get_send_comp, self.id, "send_comp(read)")?; + Ok(WorkCompletion { + wr_id: wc.wr_id, + status: CompletionStatus::from(wc.status as u32), + opcode: RdmaOp::Read, + byte_len: wc.byte_len, + imm_data: None, + }) + } + } + + pub fn device_info(&self) -> &RdmaDeviceInfo { + &self.device + } + + /// Signal the responder we are done (1-byte send) so it can tear down, then + /// disconnect. + pub fn finish(&self) { + unsafe { + let mut done = [1u8; 1]; + let mr = rdma_reg_msgs(self.id, done.as_mut_ptr().cast(), 1); + if !mr.is_null() { + if rdma_post_send(self.id, null_mut(), done.as_mut_ptr().cast(), 1, mr, 0) == 0 { + let _ = wait_comp(rdma_get_send_comp, self.id, "send_comp(done)"); + } + rdma_dereg_mr(mr); + } + rdma_disconnect(self.id); + } + } +} + +impl Drop for RealRdmaContext { + fn drop(&mut self) { + unsafe { + if !self.id.is_null() { + rdma_destroy_ep(self.id); + } + } + } +} + +/// A throwaway loopback RDMA-READ target: registers `data` for `REMOTE_READ`, +/// accepts one client, hands it `{addr, rkey, len}`, and serves until the client +/// signals done. Blocking — run on a thread; `data` must outlive the call so the +/// client's reads see real bytes. +pub struct RealResponder; + +impl RealResponder { + pub fn serve_once( + ip: &str, + port: u16, + data: &mut [u8], + ready: impl FnOnce(), + ) -> RdmaResult<()> { + unsafe { + let node = CString::new(ip).map_err(|_| op_failed("bad ip"))?; + let svc = CString::new(port.to_string()).unwrap(); + let mut hints: rdma_addrinfo = std::mem::zeroed(); + hints.ai_flags = RAI_PASSIVE as c_int; + hints.ai_port_space = rdma_port_space::RDMA_PS_TCP as c_int; + let mut res: *mut rdma_addrinfo = null_mut(); + if rdma_getaddrinfo(node.as_ptr(), svc.as_ptr(), &hints, &mut res) != 0 { + return Err(op_failed("rdma_getaddrinfo(server)")); + } + let mut attr = qp_init_attr(); + let mut listen_id: *mut rdma_cm_id = null_mut(); + let r = rdma_create_ep(&mut listen_id, res, null_mut(), &mut attr); + rdma_freeaddrinfo(res); + if r != 0 { + return Err(op_failed("rdma_create_ep(server)")); + } + if rdma_listen(listen_id, 1) != 0 { + rdma_destroy_ep(listen_id); + return Err(op_failed("rdma_listen")); + } + ready(); // the listener is up; let the client connect + + let mut id: *mut rdma_cm_id = null_mut(); + if rdma_get_request(listen_id, &mut id) != 0 { + rdma_destroy_ep(listen_id); + return Err(op_failed("rdma_get_request")); + } + + // Expose the source buffer for REMOTE_READ. + let src_mr = rdma_reg_read(id, data.as_mut_ptr().cast(), data.len()); + if src_mr.is_null() { + rdma_destroy_ep(id); + rdma_destroy_ep(listen_id); + return Err(op_failed("rdma_reg_read")); + } + + let mut meta = RemoteInfo { + addr: data.as_ptr() as u64, + rkey: (*src_mr).rkey, + len: data.len() as u32, + }; + let meta_ptr = (&mut meta as *mut RemoteInfo).cast::(); + let meta_mr = rdma_reg_msgs(id, meta_ptr, META_LEN); + let mut done = [0u8; 1]; + let done_mr = rdma_reg_msgs(id, done.as_mut_ptr().cast(), 1); + if meta_mr.is_null() || done_mr.is_null() { + return Err(op_failed("rdma_reg_msgs(server)")); + } + // Post the done-signal receive before accepting. + if rdma_post_recv(id, null_mut(), done.as_mut_ptr().cast(), 1, done_mr) != 0 { + return Err(op_failed("rdma_post_recv(done)")); + } + + let mut cp = conn_param(); + if rdma_accept(id, &mut cp) != 0 { + return Err(op_failed("rdma_accept")); + } + + // Hand the client the source MR coordinates. + if rdma_post_send(id, null_mut(), meta_ptr, META_LEN, meta_mr, 0) != 0 { + return Err(op_failed("rdma_post_send(meta)")); + } + wait_comp(rdma_get_send_comp, id, "send_comp(meta)")?; + + // Keep src_mr/data alive until the client finishes its read. + wait_comp(rdma_get_recv_comp, id, "recv_comp(done)")?; + + rdma_disconnect(id); + rdma_dereg_mr(src_mr); + rdma_dereg_mr(meta_mr); + rdma_dereg_mr(done_mr); + rdma_destroy_ep(id); + rdma_destroy_ep(listen_id); + Ok(()) + } + } +} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.cargo-ok b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.cargo-ok new file mode 100644 index 000000000..5f8b79583 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.cargo_vcs_info.json b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.cargo_vcs_info.json new file mode 100644 index 000000000..ba49c3012 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "ad39d0cba38a5ffe57c3596cfca7a4476c41b61c" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.github/workflows/ci.yml b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.github/workflows/ci.yml new file mode 100644 index 000000000..66f8e3f46 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: RDMA-SYS-CI +on: + pull_request: + branches: [master] + push: + branches: [master] + schedule: [cron: "0 */24 * * *"] + +env: + CI_RUST_TOOLCHAIN: 1.61.0 + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Git Sumbodule Update + run: | + git submodule update --remote --recursive + - name: Install dependencies + run: | + # Install libibverbs-dev as dependency automatically + sudo apt-get install -y librdmacm-dev + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ env.CI_RUST_TOOLCHAIN }} + override: true + - name: Test + run: | + ./rdma-env-setup/scripts/setup.sh + ./scripts/run.sh + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + + fmt: + name: Fmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ env.CI_RUST_TOOLCHAIN }} + override: true + - run: rustup component add rustfmt + - uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.gitignore b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.gitignore new file mode 100644 index 000000000..088ba6ba7 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.gitignore @@ -0,0 +1,10 @@ +# Generated by Cargo +# will have compiled files and executables +/target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.gitmodules b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.gitmodules new file mode 100644 index 000000000..86444c1d6 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/.gitmodules @@ -0,0 +1,4 @@ +[submodule "rdma-env-setup"] + path = rdma-env-setup + url = https://github.com/datenlord/rdma-env-setup.git + branch = main \ No newline at end of file diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.lock b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.lock new file mode 100644 index 000000000..0d30fec3b --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.lock @@ -0,0 +1,382 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bindgen" +version = "0.59.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "clap", + "env_logger", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "which", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clang-sys" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" + +[[package]] +name = "paste" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "proc-macro2" +version = "1.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rdma-sys" +version = "0.3.0" +dependencies = [ + "bindgen", + "libc", + "memoffset", + "paste", + "pkg-config", +] + +[[package]] +name = "regex" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "which" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +dependencies = [ + "either", + "libc", + "once_cell", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.toml b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.toml new file mode 100644 index 000000000..a41772dca --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.toml @@ -0,0 +1,35 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "rdma-sys" +version = "0.3.0" +authors = ["Pu Wang "] +description = "Rdma ibverbs lib Rust binding" +license-file = "LICENSE" +repository = "https://github.com/datenlord/rdma-sys" +resolver = "2" + +[dependencies.libc] +version = "0.2" + +[dependencies.memoffset] +version = "0.6" + +[dependencies.paste] +version = "1.0" + +[build-dependencies.bindgen] +version = "0.69.4" + +[build-dependencies.pkg-config] +version = "0.3" diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.toml.orig b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.toml.orig new file mode 100644 index 000000000..6c3b4adb7 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/Cargo.toml.orig @@ -0,0 +1,19 @@ +[package] +name = "rdma-sys" +version = "0.3.0" +authors = ["Pu Wang "] +edition = "2021" +description = "Rdma ibverbs lib Rust binding" +license-file = "LICENSE" +repository = "https://github.com/datenlord/rdma-sys" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libc = "0.2" +memoffset = "0.6" +paste = "1.0" +# intrusive-collections = "0.9" + +[build-dependencies] +bindgen = "0.59.2" +pkg-config = "0.3" diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/LICENSE b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/LICENSE new file mode 100644 index 000000000..d0606b7b0 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 datenlord + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/README.md b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/README.md new file mode 100644 index 000000000..17cb5abc5 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/README.md @@ -0,0 +1,3 @@ +# Rdma ibverbs lib Rust binding + +This lib is the ibverbs low-level Rust binding. As inline function and nested structure are not handled properly in the automatic bind generator, we deal with them case by case manually. diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/build.rs b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/build.rs new file mode 100644 index 000000000..ecec8afe8 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/build.rs @@ -0,0 +1,161 @@ +use std::env; +use std::path::Path; + +fn link_rdma_core(lib_name: &str, pkg_name: &str, version: &str, include_paths: &mut Vec) { + let result: _ = pkg_config::Config::new() + .atleast_version(version) + .statik(false) + .probe(lib_name); + + let lib = result.unwrap_or_else(|_| panic!("please install {pkg_name} {version})")); + println!("found {pkg_name} {}", lib.version); + + for path in lib.include_paths { + let path = path.to_str().expect("non-utf8 path"); + include_paths.push(path.to_owned()); + } +} + +fn main() { + let mut include_paths: Vec = Vec::new(); + + { + let lib_name = "libibverbs"; + let pkg_name = "libibverbs-dev"; + let version = "1.8.28"; + link_rdma_core(lib_name, pkg_name, version, &mut include_paths); + } + + { + let lib_name = "librdmacm"; + let pkg_name = "librdmacm-dev"; + let version = "1.2.28"; + link_rdma_core(lib_name, pkg_name, version, &mut include_paths); + } + + { + include_paths.sort_unstable(); + include_paths.dedup_by(|x, first| x == first); + include_paths.push("/usr/include".into()); + println!("include paths: {:?}", include_paths); + } + + let include_args = include_paths.iter().map(|p| format!("-I{}", p)); + + let bindings = bindgen::Builder::default() + .clang_args(include_args) + .header("src/bindings.h") + .allowlist_function("ibv_.*") + .allowlist_type("ibv_.*") + .allowlist_function("rdma_.*") + .allowlist_type("rdma_.*") + .allowlist_type("verbs_.*") + .allowlist_type("ib_uverbs_access_flags") + //.allowlist_type("verbs_devices_ops") + //.allowlist_var("verbs_provider_.*") + .blocklist_type("in6_addr") + .opaque_type("pthread_.*") + .blocklist_type("sockaddr.*") + .blocklist_type("timespec") + .blocklist_type("ibv_ah_attr") + .blocklist_type("ibv_async_event") + .blocklist_type("ibv_flow_spec") + .blocklist_type("ibv_gid") + .blocklist_type("ibv_global_route") + .blocklist_type("ibv_mw_bind_info") + .blocklist_type("ibv_ops_wr") + .blocklist_type("ibv_send_wr") + .blocklist_type("ibv_wc") + .blocklist_type("rdma_addr") + .blocklist_type("rdma_cm_event") + .blocklist_type("rdma_ib_addr") + .blocklist_type("rdma_ud_param") + // Following ENUM will used with bitwise-or + // including flags, mask, caps, bits, fields, size + .bitfield_enum("ibv_device_cap_flags") + .bitfield_enum("ibv_odp_transport_cap_bits") + .bitfield_enum("ibv_odp_general_caps") + .bitfield_enum("ibv_rx_hash_function_flags") + .bitfield_enum("ibv_rx_hash_fields") + .bitfield_enum("ibv_raw_packet_caps") + .bitfield_enum("ibv_tm_cap_flags") + .bitfield_enum("ibv_pci_atomic_op_size") + .bitfield_enum("ibv_port_cap_flags") + .bitfield_enum("ibv_port_cap_flags2") + .bitfield_enum("ibv_create_cq_wc_flags") + .bitfield_enum("ibv_wc_flags") + .bitfield_enum("ibv_access_flags") + .bitfield_enum("ibv_xrcd_init_attr_mask") + .bitfield_enum("ibv_rereg_mr_flags") + .bitfield_enum("ibv_srq_attr_mask") + .bitfield_enum("ibv_srq_init_attr_mask") // TODO: need to be bitfield? + .bitfield_enum("ibv_wq_init_attr_mask") + .bitfield_enum("ibv_wq_flags") + .bitfield_enum("ibv_wq_attr_mask") + .bitfield_enum("ibv_ind_table_init_attr_mask") + .bitfield_enum("ibv_qp_init_attr_mask") // TODO: need to be bitfield? + .bitfield_enum("ibv_qp_create_flags") + .bitfield_enum("ibv_qp_create_send_ops_flags") + .bitfield_enum("ibv_qp_open_attr_mask") + .bitfield_enum("ibv_qp_attr_mask") + .bitfield_enum("ibv_send_flags") + .bitfield_enum("ibv_ops_flags") + .bitfield_enum("ibv_cq_attr_mask") + .bitfield_enum("ibv_flow_flags") + .bitfield_enum("ibv_flow_action_esp_mask") + .bitfield_enum("ibv_cq_init_attr_mask") + .bitfield_enum("ibv_create_cq_attr_flags") + .bitfield_enum("ibv_parent_domain_init_attr_mask") + .bitfield_enum("ibv_read_counters_flags") + .bitfield_enum("ibv_values_mask") + .bitfield_enum("ib_uverbs_access_flags") + .bitfield_enum("rdma_cm_join_mc_attr_mask") + .bitfield_enum("rdma_cm_mc_join_flags") + // Following ENUM will be const in a sub-mod + .constified_enum_module("ibv_node_type") + .constified_enum_module("ibv_transport_type") + .constified_enum_module("ibv_atomic_cap") + .constified_enum_module("ibv_mtu") + .constified_enum_module("ibv_port_state") + .constified_enum_module("ibv_wc_status") + .constified_enum_module("ibv_wc_opcode") + .constified_enum_module("ibv_mw_type") + .constified_enum_module("ibv_rate") + .constified_enum_module("ibv_srq_type") + .constified_enum_module("ibv_wq_type") + .constified_enum_module("ibv_wq_state") + .constified_enum_module("ibv_qp_type") + .constified_enum_module("ibv_qp_state") + .constified_enum_module("ibv_mig_state") + .constified_enum_module("ibv_wr_opcode") + .constified_enum_module("ibv_ops_wr_opcode") + .constified_enum_module("ibv_flow_attr_type") + .constified_enum_module("ibv_flow_spec_type") + .constified_enum_module("ibv_counter_description") + .constified_enum_module("ibv_rereg_mr_err_code") + .constified_enum_module("ib_uverbs_advise_mr_advice") + .constified_enum_module("rdma_cm_event_type") + .constified_enum_module("rdma_driver_id") + .constified_enum_module("rdma_port_space") + .rustified_enum("ibv_event_type") + // unions with non-`Copy` fields other than `ManuallyDrop` are unstable + // for example: `pub eth: ibv_flow_spec_eth` + // note: see issue #55149 for more information + .derive_copy(true) + .derive_debug(false) + .derive_default(false) + .generate_comments(false) + //.generate_inline_functions(true) + //.default_macro_constant_type(bindgen::MacroTypeVariation::Unsigned) + .prepend_enum_name(false) + .disable_untagged_union() + .generate() + .expect("Unable to generate bindings"); + + let out_dir = env::var_os("OUT_DIR").unwrap(); + let dest_path = Path::new(&out_dir).join("bindings.rs"); + + bindings + .write_to_file(dest_path) + .expect("Could not write bindings"); +} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/examples/client.rs b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/examples/client.rs new file mode 100644 index 000000000..23ad46143 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/examples/client.rs @@ -0,0 +1,168 @@ +//! This demo shows how to establish a connection between server and client +//! and send msg to the other end. +//! +//! You can try this example by running: +//! +//! cargo run --example server +//! +//! And then start client in another terminal by running: +//! +//! cargo run --example client +//! +//! The default port is 7471. + +use rdma_sys::*; +use std::{env, process::exit, ptr::null_mut}; + +fn run(ip: &str, port: &str) -> i32 { + let mut send_msg = vec![1_u8; 16]; + let mut recv_msg = vec![0_u8; 16]; + let mut hints = unsafe { std::mem::zeroed::() }; + let mut res: *mut rdma_addrinfo = null_mut(); + + hints.ai_port_space = rdma_port_space::RDMA_PS_TCP as i32; + let mut ret = + unsafe { rdma_getaddrinfo(ip.as_ptr().cast(), port.as_ptr().cast(), &hints, &mut res) }; + + if ret != 0 { + println!("rdma_getaddrinfo"); + return ret; + } + + let mut attr = unsafe { std::mem::zeroed::() }; + let mut id: *mut rdma_cm_id = null_mut(); + attr.cap.max_send_wr = 1; + attr.cap.max_recv_wr = 1; + attr.cap.max_send_sge = 1; + attr.cap.max_recv_sge = 1; + attr.cap.max_inline_data = 16; + attr.qp_context = id.cast(); + attr.sq_sig_all = 1; + ret = unsafe { rdma_create_ep(&mut id, res, null_mut(), &mut attr) }; + // Check to see if we got inline data allowed or not + let mut send_flags = 0_u32; + if attr.cap.max_inline_data >= 16 { + send_flags = ibv_send_flags::IBV_SEND_INLINE.0; + } else { + println!("rdma_client: device doesn't support IBV_SEND_INLINE, using sge sends"); + } + + if ret != 0 { + println!("rdma_create_ep"); + unsafe { + rdma_freeaddrinfo(res); + } + return ret; + } + + let mr = unsafe { rdma_reg_msgs(id, recv_msg.as_mut_ptr().cast(), 16) }; + if mr.is_null() { + println!("rdma_reg_msgs for recv_msg"); + unsafe { + rdma_destroy_ep(id); + } + return -1; + } + + let mut send_mr = null_mut(); + if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) as u32 == 0 { + println!("flags {:?}", send_flags); + send_mr = unsafe { rdma_reg_msgs(id, send_msg.as_mut_ptr().cast(), 16) }; + if send_mr.is_null() { + println!("rdma_reg_msgs for send_msg"); + unsafe { + rdma_dereg_mr(mr); + } + return -1; + } + } + + ret = unsafe { rdma_post_recv(id, null_mut(), recv_msg.as_mut_ptr().cast(), 16, mr) }; + if ret != 0 { + println!("rdma_post_recv"); + if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) as u32 == 0 { + unsafe { rdma_dereg_mr(send_mr) }; + } + return ret; + } + + ret = unsafe { rdma_connect(id, null_mut()) }; + if ret != 0 { + println!("rdma_connect"); + unsafe { + rdma_disconnect(id); + } + return ret; + } + + ret = unsafe { + rdma_post_send( + id, + null_mut(), + send_msg.as_mut_ptr().cast(), + 16, + send_mr, + send_flags.try_into().unwrap(), + ) + }; + if ret != 0 { + println!("rdma_post_send"); + unsafe { + rdma_disconnect(id); + } + return ret; + } + + let mut wc = unsafe { std::mem::zeroed::() }; + while ret == 0 { + ret = unsafe { rdma_get_send_comp(id, &mut wc) }; + } + if ret < 0 { + println!("rdma_get_send_comp"); + unsafe { + rdma_disconnect(id); + } + return ret; + } + + ret = 0; + while ret == 0 { + ret = unsafe { rdma_get_recv_comp(id, &mut wc) }; + } + println!("rdma_client: recv msg : {:?}", recv_msg); + if ret < 0 { + println!("rdma_get_recv_comp"); + } else { + ret = 0; + } + + ret +} + +fn main() { + println!("rdma_client: start"); + let args: Vec = env::args().collect(); + if args.len() != 3 { + println!("usage : cargo run --example client "); + println!("input : {:?}", args); + exit(-1); + } + let ip = args.get(1).unwrap().as_str(); + let port = args.get(2).unwrap().as_str(); + + let ret = run(ip, port); + + if ret != 0 { + println!( + "rdma_client: ret error {:?}", + std::io::Error::from_raw_os_error(-ret) + ); + if ret == -1 { + println!( + "rdma_client: last os error {:?}", + std::io::Error::last_os_error() + ); + } + } + println!("rdma_client: end"); +} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/examples/server.rs b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/examples/server.rs new file mode 100644 index 000000000..71e8a8059 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/examples/server.rs @@ -0,0 +1,201 @@ +//! This demo shows how to establish a connection between server and client +//! and send msg to the other end. +//! +//! You can try this example by running: +//! +//! cargo run --example server +//! +//! And then start client in another terminal by running: +//! +//! cargo run --example client +//! +//! The default port is 7471. + +use rdma_sys::*; +use std::ptr::null_mut; + +static SERVER: &str = "0.0.0.0\0"; +static PORT: &str = "7471\0"; + +fn run() -> i32 { + let mut send_msg = vec![1_u8; 16]; + let mut recv_msg = vec![0_u8; 16]; + let mut hints = unsafe { std::mem::zeroed::() }; + let mut res: *mut rdma_addrinfo = null_mut(); + hints.ai_flags = RAI_PASSIVE.try_into().unwrap(); + hints.ai_port_space = rdma_port_space::RDMA_PS_TCP.try_into().unwrap(); + let mut ret = unsafe { + rdma_getaddrinfo( + SERVER.as_ptr().cast(), + PORT.as_ptr().cast(), + &hints, + &mut res, + ) + }; + + if ret != 0 { + println!("rdma_getaddrinfo"); + return ret; + } + + let mut listen_id = null_mut(); + let mut id = null_mut(); + + let mut init_attr = unsafe { std::mem::zeroed::() }; + init_attr.cap.max_send_wr = 1; + init_attr.cap.max_recv_wr = 1; + init_attr.cap.max_send_sge = 1; + init_attr.cap.max_recv_sge = 1; + init_attr.cap.max_inline_data = 16; + init_attr.sq_sig_all = 1; + ret = unsafe { rdma_create_ep(&mut listen_id, res, null_mut(), &mut init_attr) }; + // Check to see if we got inline data allowed or not + if ret != 0 { + println!("rdma_create_ep"); + unsafe { + rdma_freeaddrinfo(res); + } + return ret; + } + ret = unsafe { rdma_listen(listen_id, 0) }; + if ret != 0 { + println!("rdma_listen"); + unsafe { + rdma_destroy_ep(listen_id); + } + return ret; + } + + ret = unsafe { rdma_get_request(listen_id, &mut id) }; + if ret != 0 { + println!("rdma_get_request"); + unsafe { + rdma_destroy_ep(listen_id); + } + return ret; + } + + let mut qp_attr = unsafe { std::mem::zeroed::() }; + ret = unsafe { + ibv_query_qp( + (*id).qp, + &mut qp_attr, + ibv_qp_attr_mask::IBV_QP_CAP.0.try_into().unwrap(), + &mut init_attr, + ) + }; + + if ret != 0 { + println!("ibv_query_qp"); + unsafe { + rdma_destroy_ep(id); + } + return ret; + } + + let mut send_flags = 0_u32; + if init_attr.cap.max_inline_data >= 16 { + send_flags = ibv_send_flags::IBV_SEND_INLINE.0; + } else { + println!("rdma_server: device doesn't support IBV_SEND_INLINE, using sge sends"); + } + + let recv_mr = unsafe { rdma_reg_msgs(id, recv_msg.as_mut_ptr().cast(), 16) }; + if recv_mr.is_null() { + ret = -1; + println!("rdma_reg_msgs for recv_msg"); + unsafe { + rdma_dereg_mr(recv_mr); + } + return ret; + } + + let mut send_mr = null_mut(); + if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) == 0 { + send_mr = unsafe { rdma_reg_msgs(id, send_msg.as_mut_ptr().cast(), 16) }; + if send_mr.is_null() { + ret = -1; + println!("rdma_reg_msgs for send_msg"); + unsafe { + rdma_dereg_mr(recv_mr); + } + return ret; + } + } + ret = unsafe { rdma_post_recv(id, null_mut(), recv_msg.as_mut_ptr().cast(), 16, recv_mr) }; + + if ret != 0 { + println!("rdma_post_recv"); + unsafe { + rdma_dereg_mr(recv_mr); + } + return ret; + } + + ret = unsafe { rdma_accept(id, null_mut()) }; + if ret != 0 { + println!("rdma_accept"); + if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) == 0 { + unsafe { rdma_dereg_mr(send_mr) }; + } + return ret; + } + + let mut wc = unsafe { std::mem::zeroed::() }; + while ret == 0 { + ret = unsafe { rdma_get_recv_comp(id, &mut wc) }; + } + if ret < 0 { + println!("rdma_get_recv_comp"); + unsafe { + rdma_disconnect(id); + } + return ret; + } + println!("rdma_server: recv msg : {:?}", recv_msg); + ret = unsafe { + rdma_post_send( + id, + null_mut(), + send_msg.as_mut_ptr().cast(), + 16, + send_mr, + send_flags.try_into().unwrap(), + ) + }; + if ret != 0 { + println!("rdma_post_send"); + unsafe { + rdma_disconnect(id); + } + return ret; + } + + while ret == 0 { + ret = unsafe { rdma_get_send_comp(id, &mut wc) }; + } + if ret < 0 { + println!("rdma_get_send_comp"); + } else { + ret = 0; + } + ret +} + +fn main() { + println!("rdma_server: start"); + let ret = run(); + if ret != 0 { + println!( + "rdma_server: ret error {:?}", + std::io::Error::from_raw_os_error(-ret) + ); + if ret == -1 { + println!( + "rdma_server: last os error {:?}", + std::io::Error::last_os_error() + ); + } + } + println!("rdma_server: end"); +} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/scripts/run.sh b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/scripts/run.sh new file mode 100755 index 000000000..5b3fb4f04 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/scripts/run.sh @@ -0,0 +1,30 @@ +#! /bin/sh + +RXE_DEV=rxe_eth0 + +# Remove existing devices if any +sudo rdma link delete $RXE_DEV + +set -o errexit +set -o nounset +set -o xtrace + +if [ `ifconfig -s | grep -c '^e'` -eq 0 ]; then + echo "no eth device" + exit 1 +elif [ `ifconfig -s | grep -c '^e'` -gt 1 ]; then + echo "multiple eth devices, select the first one" + ifconfig -s | grep '^e' +fi + +ETH_DEV=`ifconfig -s | grep '^e' | cut -d ' ' -f 1 | head -n 1` +ETH_IP=`ifconfig $ETH_DEV | grep inet | grep -v inet6 | awk '{print $2}' | tr -d "addr:"` +CM_PORT=7471 +# Setup soft-roce device +sudo rdma link add $RXE_DEV type rxe netdev $ETH_DEV +rdma link | grep $RXE_DEV + +cargo test --all +cargo run --example server & +sleep 1 && +cargo run --example client $ETH_IP $CM_PORT \ No newline at end of file diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/bindings.h b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/bindings.h new file mode 100644 index 000000000..d0c264737 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/bindings.h @@ -0,0 +1,3 @@ +#include +#include +#include diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/lib.rs b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/lib.rs new file mode 100644 index 000000000..37247c1c6 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/lib.rs @@ -0,0 +1,16 @@ +#![deny(warnings)] +#![allow(non_snake_case, non_camel_case_types, non_upper_case_globals)] +#![allow(deref_nullptr)] // TODO(fxbug.dev/74605): Remove once bindgen is fixed. +#![allow(clippy::missing_safety_doc, clippy::too_many_arguments)] + +use libc::*; + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + +mod opcode; +mod types; +mod verbs; + +pub use self::opcode::*; +pub use self::types::*; +pub use self::verbs::*; diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/opcode.rs b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/opcode.rs new file mode 100644 index 000000000..cf1a1adfc --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/opcode.rs @@ -0,0 +1,103 @@ +pub mod ibv_opcode { + use paste::paste; + pub type Type = std::os::raw::c_int; + macro_rules! concat_ibv_opcode { + ($transport: expr, $op : expr ) => { + paste! { + pub const []: Type = [] + [] ; + } + }; + } + + /* transport types -- just used to define real constants */ + pub const IBV_OPCODE_RC: Type = 0x00; + pub const IBV_OPCODE_UC: Type = 0x20; + pub const IBV_OPCODE_RD: Type = 0x40; + pub const IBV_OPCODE_UD: Type = 0x60; + /* operations -- just used to define real constants */ + pub const IBV_OPCODE_SEND_FIRST: Type = 0x00; + pub const IBV_OPCODE_SEND_MIDDLE: Type = 0x01; + pub const IBV_OPCODE_SEND_LAST: Type = 0x02; + pub const IBV_OPCODE_SEND_LAST_WITH_IMMEDIATE: Type = 0x03; + pub const IBV_OPCODE_SEND_ONLY: Type = 0x04; + pub const IBV_OPCODE_SEND_ONLY_WITH_IMMEDIATE: Type = 0x05; + pub const IBV_OPCODE_RDMA_WRITE_FIRST: Type = 0x06; + pub const IBV_OPCODE_RDMA_WRITE_MIDDLE: Type = 0x07; + pub const IBV_OPCODE_RDMA_WRITE_LAST: Type = 0x08; + pub const IBV_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE: Type = 0x09; + pub const IBV_OPCODE_RDMA_WRITE_ONLY: Type = 0x0a; + pub const IBV_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE: Type = 0x0b; + pub const IBV_OPCODE_RDMA_READ_REQUEST: Type = 0x0c; + pub const IBV_OPCODE_RDMA_READ_RESPONSE_FIRST: Type = 0x0d; + pub const IBV_OPCODE_RDMA_READ_RESPONSE_MIDDLE: Type = 0x0e; + pub const IBV_OPCODE_RDMA_READ_RESPONSE_LAST: Type = 0x0f; + pub const IBV_OPCODE_RDMA_READ_RESPONSE_ONLY: Type = 0x10; + pub const IBV_OPCODE_ACKNOWLEDGE: Type = 0x11; + pub const IBV_OPCODE_ATOMIC_ACKNOWLEDGE: Type = 0x12; + pub const IBV_OPCODE_COMPARE_SWAP: Type = 0x13; + pub const IBV_OPCODE_FETCH_ADD: Type = 0x14; + + /* RC */ + concat_ibv_opcode!(RC, SEND_FIRST); + concat_ibv_opcode!(RC, SEND_MIDDLE); + concat_ibv_opcode!(RC, SEND_LAST); + concat_ibv_opcode!(RC, SEND_LAST_WITH_IMMEDIATE); + concat_ibv_opcode!(RC, SEND_ONLY); + concat_ibv_opcode!(RC, SEND_ONLY_WITH_IMMEDIATE); + concat_ibv_opcode!(RC, RDMA_WRITE_FIRST); + concat_ibv_opcode!(RC, RDMA_WRITE_MIDDLE); + concat_ibv_opcode!(RC, RDMA_WRITE_LAST); + concat_ibv_opcode!(RC, RDMA_WRITE_LAST_WITH_IMMEDIATE); + concat_ibv_opcode!(RC, RDMA_WRITE_ONLY); + concat_ibv_opcode!(RC, RDMA_WRITE_ONLY_WITH_IMMEDIATE); + concat_ibv_opcode!(RC, RDMA_READ_REQUEST); + concat_ibv_opcode!(RC, RDMA_READ_RESPONSE_FIRST); + concat_ibv_opcode!(RC, RDMA_READ_RESPONSE_MIDDLE); + concat_ibv_opcode!(RC, RDMA_READ_RESPONSE_LAST); + concat_ibv_opcode!(RC, RDMA_READ_RESPONSE_ONLY); + concat_ibv_opcode!(RC, ACKNOWLEDGE); + concat_ibv_opcode!(RC, ATOMIC_ACKNOWLEDGE); + concat_ibv_opcode!(RC, COMPARE_SWAP); + concat_ibv_opcode!(RC, FETCH_ADD); + + /* UC */ + concat_ibv_opcode!(UC, SEND_FIRST); + concat_ibv_opcode!(UC, SEND_MIDDLE); + concat_ibv_opcode!(UC, SEND_LAST); + concat_ibv_opcode!(UC, SEND_LAST_WITH_IMMEDIATE); + concat_ibv_opcode!(UC, SEND_ONLY); + concat_ibv_opcode!(UC, SEND_ONLY_WITH_IMMEDIATE); + concat_ibv_opcode!(UC, RDMA_WRITE_FIRST); + concat_ibv_opcode!(UC, RDMA_WRITE_MIDDLE); + concat_ibv_opcode!(UC, RDMA_WRITE_LAST); + concat_ibv_opcode!(UC, RDMA_WRITE_LAST_WITH_IMMEDIATE); + concat_ibv_opcode!(UC, RDMA_WRITE_ONLY); + concat_ibv_opcode!(UC, RDMA_WRITE_ONLY_WITH_IMMEDIATE); + + /* RD */ + concat_ibv_opcode!(RD, SEND_FIRST); + concat_ibv_opcode!(RD, SEND_MIDDLE); + concat_ibv_opcode!(RD, SEND_LAST); + concat_ibv_opcode!(RD, SEND_LAST_WITH_IMMEDIATE); + concat_ibv_opcode!(RD, SEND_ONLY); + concat_ibv_opcode!(RD, SEND_ONLY_WITH_IMMEDIATE); + concat_ibv_opcode!(RD, RDMA_WRITE_FIRST); + concat_ibv_opcode!(RD, RDMA_WRITE_MIDDLE); + concat_ibv_opcode!(RD, RDMA_WRITE_LAST); + concat_ibv_opcode!(RD, RDMA_WRITE_LAST_WITH_IMMEDIATE); + concat_ibv_opcode!(RD, RDMA_WRITE_ONLY); + concat_ibv_opcode!(RD, RDMA_WRITE_ONLY_WITH_IMMEDIATE); + concat_ibv_opcode!(RD, RDMA_READ_REQUEST); + concat_ibv_opcode!(RD, RDMA_READ_RESPONSE_FIRST); + concat_ibv_opcode!(RD, RDMA_READ_RESPONSE_MIDDLE); + concat_ibv_opcode!(RD, RDMA_READ_RESPONSE_LAST); + concat_ibv_opcode!(RD, RDMA_READ_RESPONSE_ONLY); + concat_ibv_opcode!(RD, ACKNOWLEDGE); + concat_ibv_opcode!(RD, ATOMIC_ACKNOWLEDGE); + concat_ibv_opcode!(RD, COMPARE_SWAP); + concat_ibv_opcode!(RD, FETCH_ADD); + + /* UD */ + concat_ibv_opcode!(UD, SEND_ONLY); + concat_ibv_opcode!(UD, SEND_ONLY_WITH_IMMEDIATE); +} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/types.rs b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/types.rs new file mode 100644 index 000000000..f1f6e35a4 --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/types.rs @@ -0,0 +1,299 @@ +use crate::*; + +/// This file defines the types directly or indirectly involving union, +/// in that BindGen cannot handle union very well, so mannually define them. + +/// Struct types involve union in + +// ibv_gid related union and struct types +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ibv_gid_global_t { + pub subnet_prefix: __be64, + pub interface_id: __be64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub union ibv_gid { + pub raw: [u8; 16], + pub global: ibv_gid_global_t, +} + +// ibv_async_event related union and struct type +#[repr(C)] +pub union ibv_async_event_element_t { + pub cq: *mut ibv_cq, + pub qp: *mut ibv_qp, + pub srq: *mut ibv_srq, + pub wq: *mut ibv_wq, + pub port_num: c_int, +} + +#[repr(C)] +pub struct ibv_async_event { + pub element: ibv_async_event_element_t, + pub event_type: ibv_event_type, +} + +// ibv_wc related union and struct types +#[repr(C)] +pub union imm_data_invalidated_rkey_union_t { + /// When (wc_flags & IBV_WC_WITH_IMM): Immediate data in network byte order. + pub imm_data: __be32, + /// When (wc_flags & IBV_WC_WITH_INV): Stores the invalidated rkey. + pub invalidated_rkey: u32, +} + +#[repr(C)] +pub struct ibv_wc { + pub wr_id: u64, + pub status: ibv_wc_status::Type, + pub opcode: ibv_wc_opcode::Type, + pub vendor_err: u32, + pub byte_len: u32, + pub imm_data_invalidated_rkey_union: imm_data_invalidated_rkey_union_t, + pub qp_num: u32, + pub src_qp: u32, + pub wc_flags: c_uint, + pub pkey_index: u16, + pub slid: u16, + pub sl: u8, + pub dlid_path_bits: u8, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ibv_global_route { + pub dgid: ibv_gid, + pub flow_label: u32, + pub sgid_index: u8, + pub hop_limit: u8, + pub traffic_class: u8, +} + +// ibv_send_wr related union and struct types +#[repr(C)] +#[derive(Copy, Clone)] +pub struct ibv_mw_bind_info { + pub mr: *mut ibv_mr, + pub addr: u64, + pub length: u64, + pub mw_access_flags: ::std::os::raw::c_uint, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct rdma_t { + pub remote_addr: u64, + pub rkey: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct atomic_t { + pub remote_addr: u64, + pub compare_add: u64, + pub swap: u64, + pub rkey: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ud_t { + pub ah: *mut ibv_ah, + pub remote_qpn: u32, + pub remote_qkey: u32, +} + +#[repr(C)] +pub union wr_t { + pub rdma: rdma_t, + pub atomic: atomic_t, + pub ud: ud_t, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct xrc_t { + pub remote_srqn: u32, +} + +#[repr(C)] +pub union qp_type_t { + pub xrc: xrc_t, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct bind_mw_t { + pub mw: *mut ibv_mw, + pub rkey: u32, + pub bind_info: ibv_mw_bind_info, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct tso_t { + pub hdr: *mut c_void, + pub hdr_sz: u16, + pub mss: u16, +} + +#[repr(C)] +pub union bind_mw_tso_union_t { + pub bind_mw: bind_mw_t, + pub tso: tso_t, +} + +#[repr(C)] +pub struct ibv_send_wr { + pub wr_id: u64, + pub next: *mut Self, + pub sg_list: *mut ibv_sge, + pub num_sge: c_int, + pub opcode: ibv_wr_opcode::Type, + pub send_flags: c_uint, + /// When opcode is *_WITH_IMM: Immediate data in network byte order. + /// When opcode is *_INV: Stores the rkey to invalidate + pub imm_data_invalidated_rkey_union: imm_data_invalidated_rkey_union_t, + pub wr: wr_t, + pub qp_type: qp_type_t, + pub bind_mw_tso_union: bind_mw_tso_union_t, +} + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct add_t { + pub recv_wr_id: u64, + pub sg_list: *mut ibv_sge, + pub num_sge: c_int, + pub tag: u64, + pub mask: u64, +} + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct tm_t { + pub unexpected_cnt: u32, + pub handle: u32, + pub add: add_t, +} + +#[repr(C)] +pub struct ibv_ops_wr { + wr_id: u64, + next: *mut Self, + opcode: ibv_ops_wr_opcode::Type, + flags: c_int, + tm: tm_t, +} + +// ibv_flow_spec related union and struct types +#[repr(C)] +#[derive(Clone, Copy)] +pub struct hdr_t { + pub type_: ibv_flow_spec_type::Type, + pub size: u16, +} + +#[repr(C)] +pub union ibv_flow_spec_union_t { + pub hdr: hdr_t, + pub eth: ibv_flow_spec_eth, + pub ipv4: ibv_flow_spec_ipv4, + pub tcp_udp: ibv_flow_spec_tcp_udp, + pub ipv4_ext: ibv_flow_spec_ipv4_ext, + pub ipv6: ibv_flow_spec_ipv6, + pub esp: ibv_flow_spec_esp, + pub tunnel: ibv_flow_spec_tunnel, + pub gre: ibv_flow_spec_gre, + pub mpls: ibv_flow_spec_mpls, + pub flow_tag: ibv_flow_spec_action_tag, + pub drop: ibv_flow_spec_action_drop, + pub handle: ibv_flow_spec_action_handle, + pub flow_count: ibv_flow_spec_counter_action, +} + +#[repr(C)] +pub struct ibv_flow_spec { + pub ibv_flow_spec_union: ibv_flow_spec_union_t, +} + +/// Struct types involve union in + +// rdma_addr related union and struct types +#[repr(C)] +#[derive(Clone, Copy)] +pub struct rdma_ib_addr { + pub sgid: ibv_gid, + pub dgid: ibv_gid, + pub pkey: __be16, +} + +#[repr(C)] +pub union src_addr_union_t { + pub src_addr: libc::sockaddr, + pub src_sin: libc::sockaddr_in, + pub src_sin6: libc::sockaddr_in6, + pub src_storage: libc::sockaddr_storage, +} + +#[repr(C)] +pub union dst_addr_union_t { + pub dst_addr: libc::sockaddr, + pub dst_sin: libc::sockaddr_in, + pub dst_sin6: libc::sockaddr_in6, + pub dst_storage: libc::sockaddr_storage, +} + +#[repr(C)] +pub union addr_union_t { + pub ibaddr: rdma_ib_addr, +} + +#[repr(C)] +pub struct rdma_addr { + pub src_addr_union: src_addr_union_t, + pub dst_addr_union: dst_addr_union_t, + pub addr: addr_union_t, +} + +/// rdma_cm_event related union and struct types + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ibv_ah_attr { + pub grh: ibv_global_route, + pub dlid: u16, + pub sl: u8, + pub src_path_bits: u8, + pub static_rate: u8, + pub is_global: u8, + pub port_num: u8, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct rdma_ud_param { + pub private_data: *const ::std::os::raw::c_void, + pub private_data_len: u8, + pub ah_attr: ibv_ah_attr, + pub qp_num: u32, + pub qkey: u32, +} + +#[repr(C)] +pub union param_t { + pub conn: rdma_conn_param, + pub ud: rdma_ud_param, +} + +#[repr(C)] +pub struct rdma_cm_event { + pub id: *mut rdma_cm_id, + pub listen_id: *mut rdma_cm_id, + pub event: rdma_cm_event_type::Type, + pub status: c_int, + pub param: param_t, +} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/verbs.rs b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/verbs.rs new file mode 100644 index 000000000..86c9ed41c --- /dev/null +++ b/seaweedfs-rdma-sidecar/rdma-engine/vendor/rdma-sys/src/verbs.rs @@ -0,0 +1,1367 @@ +use crate::*; + +use std::mem; +use std::ptr; + +/// Inline functions from + +pub type ibv_advise_mr_advice = ib_uverbs_advise_mr_advice::Type; + +// ibv_qp_ex related inline functions +#[inline] +pub unsafe fn ibv_wr_atomic_cmp_swp( + qp: *mut ibv_qp_ex, + rkey: u32, + remote_addr: u64, + compare: u64, + swap: u64, +) { + (*qp).wr_atomic_cmp_swp.unwrap()(qp, rkey, remote_addr, compare, swap); +} + +#[inline] +pub unsafe fn ibv_wr_atomic_fetch_add(qp: *mut ibv_qp_ex, rkey: u32, remote_addr: u64, add: u64) { + (*qp).wr_atomic_fetch_add.unwrap()(qp, rkey, remote_addr, add); +} + +#[inline] +pub unsafe fn ibv_wr_bind_mw( + qp: *mut ibv_qp_ex, + mw: *mut ibv_mw, + rkey: u32, + bind_info: *const ibv_mw_bind_info, +) { + (*qp).wr_bind_mw.unwrap()(qp, mw, rkey, bind_info); +} + +#[inline] +pub unsafe fn ibv_wr_local_inv(qp: *mut ibv_qp_ex, invalidate_rkey: u32) { + (*qp).wr_local_inv.unwrap()(qp, invalidate_rkey); +} + +#[inline] +pub unsafe fn ibv_wr_rdma_read(qp: *mut ibv_qp_ex, rkey: u32, remote_addr: u64) { + (*qp).wr_rdma_read.unwrap()(qp, rkey, remote_addr); +} + +#[inline] +pub unsafe fn ibv_wr_rdma_write(qp: *mut ibv_qp_ex, rkey: u32, remote_addr: u64) { + (*qp).wr_rdma_write.unwrap()(qp, rkey, remote_addr); +} + +#[inline] +pub unsafe fn ibv_wr_rdma_write_imm( + qp: *mut ibv_qp_ex, + rkey: u32, + remote_addr: u64, + imm_data: __be32, +) { + (*qp).wr_rdma_write_imm.unwrap()(qp, rkey, remote_addr, imm_data); +} + +#[inline] +pub unsafe fn ibv_wr_send(qp: *mut ibv_qp_ex) { + (*qp).wr_send.unwrap()(qp); +} + +#[inline] +pub unsafe fn ibv_wr_send_imm(qp: *mut ibv_qp_ex, imm_data: __be32) { + (*qp).wr_send_imm.unwrap()(qp, imm_data); +} + +#[inline] +pub unsafe fn ibv_wr_send_inv(qp: *mut ibv_qp_ex, invalidate_rkey: u32) { + (*qp).wr_send_inv.unwrap()(qp, invalidate_rkey); +} + +#[inline] +pub unsafe fn ibv_wr_send_tso(qp: *mut ibv_qp_ex, hdr: *mut c_void, hdr_sz: u16, mss: u16) { + (*qp).wr_send_tso.unwrap()(qp, hdr, hdr_sz, mss); +} + +#[inline] +pub unsafe fn ibv_wr_set_ud_addr( + qp: *mut ibv_qp_ex, + ah: *mut ibv_ah, + remote_qpn: u32, + remote_qkey: u32, +) { + (*qp).wr_set_ud_addr.unwrap()(qp, ah, remote_qpn, remote_qkey); +} + +#[inline] +pub unsafe fn ibv_wr_set_xrc_srqn(qp: *mut ibv_qp_ex, remote_srqn: u32) { + (*qp).wr_set_xrc_srqn.unwrap()(qp, remote_srqn); +} + +#[inline] +pub unsafe fn ibv_wr_set_inline_data(qp: *mut ibv_qp_ex, addr: *mut c_void, length: usize) { + (*qp).wr_set_inline_data.unwrap()(qp, addr, length); +} + +#[inline] +pub unsafe fn ibv_wr_set_inline_data_list( + qp: *mut ibv_qp_ex, + num_buf: usize, + buf_list: *const ibv_data_buf, +) { + (*qp).wr_set_inline_data_list.unwrap()(qp, num_buf, buf_list); +} + +#[inline] +pub unsafe fn ibv_wr_set_sge(qp: *mut ibv_qp_ex, lkey: u32, addr: u64, length: u32) { + (*qp).wr_set_sge.unwrap()(qp, lkey, addr, length); +} + +#[inline] +pub unsafe fn ibv_wr_set_sge_list(qp: *mut ibv_qp_ex, num_sge: usize, sg_list: *const ibv_sge) { + (*qp).wr_set_sge_list.unwrap()(qp, num_sge, sg_list); +} + +#[inline] +pub unsafe fn ibv_wr_start(qp: *mut ibv_qp_ex) { + (*qp).wr_start.unwrap()(qp); +} + +#[inline] +pub unsafe fn ibv_wr_complete(qp: *mut ibv_qp_ex) -> c_int { + (*qp).wr_complete.unwrap()(qp) +} + +#[inline] +pub unsafe fn ibv_wr_abort(qp: *mut ibv_qp_ex) { + (*qp).wr_abort.unwrap()(qp) +} + +// ibv_cq_ex related inline functions +#[inline] +pub unsafe fn ibv_cq_ex_to_cq(cq: *mut ibv_cq_ex) -> *mut ibv_cq { + cq as *mut ibv_cq_ex as *mut ibv_cq +} + +#[inline] +pub unsafe fn ibv_start_poll(cq: *mut ibv_cq_ex, attr: *mut ibv_poll_cq_attr) -> c_int { + (*cq).start_poll.unwrap()(cq, attr) +} + +#[inline] +pub unsafe fn ibv_next_poll(cq: *mut ibv_cq_ex) -> c_int { + (*cq).next_poll.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_end_poll(cq: *mut ibv_cq_ex) { + (*cq).end_poll.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_opcode(cq: *mut ibv_cq_ex) -> ibv_wc_opcode::Type { + (*cq).read_opcode.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_vendor_err(cq: *mut ibv_cq_ex) -> u32 { + (*cq).read_vendor_err.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_byte_len(cq: *mut ibv_cq_ex) -> u32 { + (*cq).read_byte_len.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_imm_data(cq: *mut ibv_cq_ex) -> __be32 { + (*cq).read_imm_data.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_invalidated_rkey(cq: *mut ibv_cq_ex) -> u32 { + // #ifdef __CHECKER__ + // return (__attribute__((force)) uint32_t)cq->read_imm_data(cq); + // #else + // return cq->read_imm_data(cq); + // #endif + (*cq).read_imm_data.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_qp_num(cq: *mut ibv_cq_ex) -> u32 { + (*cq).read_qp_num.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_src_qp(cq: *mut ibv_cq_ex) -> u32 { + (*cq).read_src_qp.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_wc_flags(cq: *mut ibv_cq_ex) -> c_uint { + (*cq).read_wc_flags.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_slid(cq: *mut ibv_cq_ex) -> u32 { + (*cq).read_slid.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_sl(cq: *mut ibv_cq_ex) -> u8 { + (*cq).read_sl.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_dlid_path_bits(cq: *mut ibv_cq_ex) -> u8 { + (*cq).read_dlid_path_bits.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_completion_ts(cq: *mut ibv_cq_ex) -> u64 { + (*cq).read_completion_ts.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_completion_wallclock_ns(cq: *mut ibv_cq_ex) -> u64 { + (*cq).read_completion_wallclock_ns.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_cvlan(cq: *mut ibv_cq_ex) -> u16 { + (*cq).read_cvlan.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_flow_tag(cq: *mut ibv_cq_ex) -> u32 { + (*cq).read_flow_tag.unwrap()(cq) +} + +#[inline] +pub unsafe fn ibv_wc_read_tm_info(cq: *mut ibv_cq_ex, tm_info: *mut ibv_wc_tm_info) { + (*cq).read_tm_info.unwrap()(cq, tm_info) +} + +// ibv_wq related inline function +#[inline] +pub unsafe fn ibv_post_wq_recv( + wq: *mut ibv_wq, + recv_wr: *mut ibv_recv_wr, + bad_recv_wr: *mut *mut ibv_recv_wr, +) -> c_int { + (*wq).post_recv.unwrap()(wq, recv_wr, bad_recv_wr) +} + +// Use intrusive_collections::container_of! instread, once it's stable not nightly +macro_rules! container_of { + ($ptr:expr, $container:path, $field:ident) => {{ + ($ptr as *const _ as *const u8).sub(memoffset::offset_of!($container, $field)) + as *const $container + }}; +} + +// Utility function to get verbs_context from ibv_context +#[inline] +unsafe fn verbs_get_ctx(ctx: *const ibv_context) -> Option<*mut verbs_context> { + if (*ctx).abi_compat as usize != u32::MAX as usize { + None + } else { + let vcp = container_of!(ctx, verbs_context, context) as *mut _; + Some(vcp) + } +} + +macro_rules! verbs_get_ctx_op { + ($vcr:expr, $field:ident) => { + if let Some(vc) = verbs_get_ctx($vcr) { + if (*vc).sz < mem::size_of_val(&*vc) - memoffset::offset_of!(verbs_context, $field) { + None + } else { + if (*vc).$field.is_some() { + Some(vc) + } else { + None + } + } + } else { + None + } + }; +} + +// TODO: note that ibv_query_port, ibv_get_device_list, ibv_reg_mr, and +// ibv_reg_mr_iova are redefined using ___ibv_query_port, +// __ibv_get_device_list, __ibv_reg_mr, and __ibv_reg_mr_iova in C, which +// should be handled properly in Rust. + +// When statically linking the user can set RDMA_STATIC_PROVIDERS to a comma +// separated list of provider names to include in the static link, and this +// machinery will cause those providers to be included statically. +// +// Linking will fail if this is set for dynamic linking. +// +// #define ibv_get_device_list(num_devices) __ibv_get_device_list(num_devices) +// #endif +// #ifdef RDMA_STATIC_PROVIDERS +// #define _RDMA_STATIC_PREFIX_(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, \ +// _12, _13, _14, _15, _16, _17, ...) \ +// &verbs_provider_##_1, &verbs_provider_##_2, &verbs_provider_##_3, \ +// &verbs_provider_##_4, &verbs_provider_##_5, \ +// &verbs_provider_##_6, &verbs_provider_##_7, \ +// &verbs_provider_##_8, &verbs_provider_##_9, \ +// &verbs_provider_##_10, &verbs_provider_##_11, \ +// &verbs_provider_##_12, &verbs_provider_##_13, \ +// &verbs_provider_##_14, &verbs_provider_##_15, \ +// &verbs_provider_##_16, &verbs_provider_##_17 +// #define _RDMA_STATIC_PREFIX(arg) \ +// _RDMA_STATIC_PREFIX_(arg, none, none, none, none, none, none, none, \ +// none, none, none, none, none, none, none, none, \ +// none) +pub unsafe fn __ibv_get_device_list(num_devices: *mut c_int) -> *mut *mut ibv_device { + // TODO: check static linking compatibility + // ibv_static_providers(NULL, _RDMA_STATIC_PREFIX(RDMA_STATIC_PROVIDERS), NULL); + ibv_get_device_list(num_devices) +} + +// TODO: missing variable args function +// void ibv_static_providers(void *unused, ...); + +// ibv_context related inline function +#[inline] +pub unsafe fn ___ibv_query_port( + context: *mut ibv_context, + port_num: u8, + port_attr: *mut ibv_port_attr, +) -> c_int { + let vcr = verbs_get_ctx_op!(context, query_port); + + if let Some(vctx) = vcr { + (*vctx).query_port.unwrap()(context, port_num, port_attr, mem::size_of_val(&*port_attr)) + } else { + // TODO: memset(port_attr, 0, sizeof(*port_attr)); + let compat_attr = port_attr as *mut _ as *mut _compat_ibv_port_attr; + ibv_query_port(context, port_num, compat_attr) + } +} + +// ibv_flow related inline functions +#[inline] +pub unsafe fn ibv_create_flow(qp: *mut ibv_qp, flow: *mut ibv_flow_attr) -> Option<*mut ibv_flow> { + let vcr = verbs_get_ctx_op!((*qp).context, ibv_create_flow); + + if let Some(vctx) = vcr { + Some((*vctx).ibv_create_flow.unwrap()(qp, flow)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_destroy_flow(flow_id: *mut ibv_flow) -> c_int { + let vcr = verbs_get_ctx_op!((*flow_id).context, ibv_destroy_flow); + + if let Some(vctx) = vcr { + (*vctx).ibv_destroy_flow.unwrap()(flow_id) + } else { + libc::EOPNOTSUPP + } +} + +#[inline] +pub unsafe fn ibv_create_flow_action_esp( + ctx: *mut ibv_context, + esp: *mut ibv_flow_action_esp_attr, +) -> Option<*mut ibv_flow_action> { + let vcr = verbs_get_ctx_op!(ctx, create_flow_action_esp); + + if let Some(vctx) = vcr { + Some((*vctx).create_flow_action_esp.unwrap()(ctx, esp)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_modify_flow_action_esp( + action: *mut ibv_flow_action, + esp: *mut ibv_flow_action_esp_attr, +) -> c_int { + let vcr = verbs_get_ctx_op!((*action).context, modify_flow_action_esp); + + if let Some(vctx) = vcr { + (*vctx).modify_flow_action_esp.unwrap()(action, esp) + } else { + libc::EOPNOTSUPP + } +} + +#[inline] +pub unsafe fn ibv_destroy_flow_action(action: *mut ibv_flow_action) -> c_int { + let vcr = verbs_get_ctx_op!((*action).context, destroy_flow_action); + + if let Some(vctx) = vcr { + (*vctx).destroy_flow_action.unwrap()(action) + } else { + libc::EOPNOTSUPP + } +} + +// ibv_xrcd related inline functions +#[inline] +pub unsafe fn ibv_open_xrcd( + context: *mut ibv_context, + xrcd_init_attr: *mut ibv_xrcd_init_attr, +) -> Option<*mut ibv_xrcd> { + let vcr = verbs_get_ctx_op!(context, open_xrcd); + + if let Some(vctx) = vcr { + Some((*vctx).open_xrcd.unwrap()(context, xrcd_init_attr)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_close_xrcd(xrcd: *mut ibv_xrcd) -> c_int { + let vctx = verbs_get_ctx((*xrcd).context); + + (*vctx.unwrap()).close_xrcd.unwrap()(xrcd) +} + +// use new ibv_reg_mr version only if access flags that require it are used +#[inline] +pub unsafe fn __ibv_reg_mr( + pd: *mut ibv_pd, + addr: *mut c_void, + length: usize, + access: c_uint, + is_access_const: c_int, +) -> *mut ibv_mr { + if is_access_const != 0 + && (ib_uverbs_access_flags(access) + & ib_uverbs_access_flags::IB_UVERBS_ACCESS_OPTIONAL_RANGE) + == ib_uverbs_access_flags(0) + { + ibv_reg_mr(pd, addr, length, access as c_int) + } else { + ibv_reg_mr_iova2(pd, addr, length, addr as u64, access) + } +} +// TODO: handle C macro defined function +// #define ibv_reg_mr(pd, addr, length, access) \ +// __ibv_reg_mr(pd, addr, length, access, \ +// __builtin_constant_p( \ +// ((access) & IBV_ACCESS_OPTIONAL_RANGE) == 0)) + +// use new ibv_reg_mr version only if access flags that require it are used +#[inline] +pub unsafe fn __ibv_reg_mr_iova( + pd: *mut ibv_pd, + addr: *mut c_void, + length: usize, + iova: u64, + access: c_uint, + is_access_const: c_int, +) -> *mut ibv_mr { + if is_access_const != 0 + && (ib_uverbs_access_flags(access) + & ib_uverbs_access_flags::IB_UVERBS_ACCESS_OPTIONAL_RANGE) + == ib_uverbs_access_flags(0) + { + ibv_reg_mr_iova(pd, addr, length, iova, access as c_int) + } else { + ibv_reg_mr_iova2(pd, addr, length, iova, access) + } +} +// TODO: handle C macro defined function +// #define ibv_reg_mr_iova(pd, addr, length, iova, access) \ +// __ibv_reg_mr_iova(pd, addr, length, iova, access, \ +// __builtin_constant_p( \ +// ((access) & IBV_ACCESS_OPTIONAL_RANGE) == 0)) + +// ibv_mw related inline functions +#[inline] +pub unsafe fn ibv_alloc_mw(pd: *mut ibv_pd, type_: ibv_mw_type::Type) -> Option<*mut ibv_mw> { + if (*(*pd).context).ops.alloc_mw.is_some() { + Some((*(*pd).context).ops.alloc_mw.unwrap()(pd, type_)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_dealloc_mw(mw: *mut ibv_mw) -> c_int { + (*(*mw).context).ops.dealloc_mw.unwrap()(mw) +} + +// ibv_inc_rkey - Increase the 8 lsb in the given rkey +#[inline] +pub unsafe fn ibv_inc_rkey(rkey: u32) -> u32 { + let mask: u32 = 0x000000ff; + let newtag = ((rkey + 1) & mask) as u8; + + (rkey & !mask) | (newtag as u32) +} + +#[inline] +pub unsafe fn ibv_bind_mw(qp: *mut ibv_qp, mw: *mut ibv_mw, mw_bind: *mut ibv_mw_bind) -> c_int { + if (*mw).type_ != ibv_mw_type::IBV_MW_TYPE_1 { + libc::EINVAL + } else { + (*(*mw).context).ops.bind_mw.unwrap()(qp, mw, mw_bind) + } +} + +#[inline] +pub unsafe fn ibv_advise_mr( + pd: *mut ibv_pd, + advice: ibv_advise_mr_advice, + flags: u32, + sg_list: *mut ibv_sge, + num_sge: u32, +) -> c_int { + let vcr = verbs_get_ctx_op!((*pd).context, advise_mr); + + if let Some(vctx) = vcr { + (*vctx).advise_mr.unwrap()(pd, advice, flags, sg_list, num_sge) + } else { + libc::EOPNOTSUPP + } +} + +// ibv_dm related inline functions +#[inline] +pub unsafe fn ibv_alloc_dm( + context: *mut ibv_context, + attr: *mut ibv_alloc_dm_attr, +) -> Option<*mut ibv_dm> { + let vcr = verbs_get_ctx_op!(context, alloc_dm); + + if let Some(vctx) = vcr { + Some((*vctx).alloc_dm.unwrap()(context, attr)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_free_dm(dm: *mut ibv_dm) -> c_int { + let vcr = verbs_get_ctx_op!((*dm).context, free_dm); + + if let Some(vctx) = vcr { + (*vctx).free_dm.unwrap()(dm) + } else { + libc::EOPNOTSUPP + } +} + +#[inline] +pub unsafe fn ibv_memcpy_to_dm( + dm: *mut ibv_dm, + dm_offset: u64, + host_addr: *const c_void, + length: usize, +) -> c_int { + (*dm).memcpy_to_dm.unwrap()(dm, dm_offset, host_addr, length) +} + +#[inline] +pub unsafe fn ibv_memcpy_from_dm( + host_addr: *mut c_void, + dm: *mut ibv_dm, + dm_offset: u64, + length: usize, +) -> c_int { + (*dm).memcpy_from_dm.unwrap()(host_addr, dm, dm_offset, length) +} + +#[inline] +pub unsafe fn ibv_alloc_null_mr(pd: *mut ibv_pd) -> Option<*mut ibv_mr> { + let vcr = verbs_get_ctx_op!((*pd).context, alloc_null_mr); + + if let Some(vctx) = vcr { + Some((*vctx).alloc_null_mr.unwrap()(pd)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_reg_dm_mr( + pd: *mut ibv_pd, + dm: *mut ibv_dm, + dm_offset: u64, + length: usize, + access: u32, +) -> Option<*mut ibv_mr> { + let vcr = verbs_get_ctx_op!((*pd).context, reg_dm_mr); + + if let Some(vctx) = vcr { + Some((*vctx).reg_dm_mr.unwrap()( + pd, dm, dm_offset, length, access, + )) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +// ibv_cq_ex related inline function +#[inline] +pub unsafe fn ibv_create_cq_ex( + context: *mut ibv_context, + cq_attr: *mut ibv_cq_init_attr_ex, +) -> Option<*mut ibv_cq_ex> { + let vcr = verbs_get_ctx_op!(context, create_cq_ex); + + if let Some(vctx) = vcr { + Some((*vctx).create_cq_ex.unwrap()(context, cq_attr)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +// ibv_cq related inline functions +#[inline] +pub unsafe fn ibv_poll_cq(cq: *mut ibv_cq, num_entries: i32, wc: *mut ibv_wc) -> c_int { + (*(*cq).context).ops.poll_cq.unwrap()(cq, num_entries, wc) +} + +#[inline] +pub unsafe fn ibv_req_notify_cq(cq: *mut ibv_cq, solicited_only: i32) -> c_int { + (*(*cq).context).ops.req_notify_cq.unwrap()(cq, solicited_only) +} + +#[inline] +pub unsafe fn ibv_modify_cq(cq: *mut ibv_cq, attr: *mut ibv_modify_cq_attr) -> c_int { + let vcr = verbs_get_ctx_op!((*cq).context, modify_cq); + + if let Some(vctx) = vcr { + (*vctx).modify_cq.unwrap()(cq, attr) + } else { + libc::EOPNOTSUPP + } +} + +// ibv_srq related inline functions +#[inline] +pub unsafe fn ibv_create_srq_ex( + context: *mut ibv_context, + srq_init_attr_ex: *mut ibv_srq_init_attr_ex, +) -> Option<*mut ibv_srq> { + let mask = ibv_srq_init_attr_mask((*srq_init_attr_ex).comp_mask); + let mask_inv = ibv_srq_init_attr_mask(!(*srq_init_attr_ex).comp_mask); + let zero = ibv_srq_init_attr_mask(0); + + // TODO: verify the condition + let cond = (mask_inv + | (ibv_srq_init_attr_mask::IBV_SRQ_INIT_ATTR_PD + & ibv_srq_init_attr_mask::IBV_SRQ_INIT_ATTR_TYPE)) + != zero + && (mask & ibv_srq_init_attr_mask::IBV_SRQ_INIT_ATTR_PD) != zero + && ((mask & ibv_srq_init_attr_mask::IBV_SRQ_INIT_ATTR_TYPE) != zero + || ((*srq_init_attr_ex).srq_type == ibv_srq_type::IBV_SRQT_BASIC)); + if cond { + Some(ibv_create_srq( + (*srq_init_attr_ex).pd, + srq_init_attr_ex as *mut ibv_srq_init_attr, + )) + } else { + let vcr = verbs_get_ctx_op!(context, create_srq_ex); + + if let Some(vctx) = vcr { + Some((*vctx).create_srq_ex.unwrap()(context, srq_init_attr_ex)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } + } +} + +#[inline] +pub unsafe fn ibv_get_srq_num(srq: *mut ibv_srq, srq_num: *mut u32) -> c_int { + let vcr = verbs_get_ctx_op!((*srq).context, get_srq_num); + + if let Some(vctx) = vcr { + (*vctx).get_srq_num.unwrap()(srq, srq_num) + } else { + libc::EOPNOTSUPP + } +} + +#[inline] +pub unsafe fn ibv_post_srq_recv( + srq: *mut ibv_srq, + recv_wr: *mut ibv_recv_wr, + bad_recv_wr: *mut *mut ibv_recv_wr, +) -> c_int { + (*(*srq).context).ops.post_srq_recv.unwrap()(srq, recv_wr, bad_recv_wr) +} + +#[inline] +pub unsafe fn ibv_post_srq_ops( + srq: *mut ibv_srq, + op: *mut ibv_ops_wr, + bad_op: *mut *mut ibv_ops_wr, +) -> c_int { + let vcr = verbs_get_ctx_op!((*srq).context, post_srq_ops); + + if let Some(vctx) = vcr { + (*vctx).post_srq_ops.unwrap()(srq, op, bad_op) + } else { + *bad_op = op; + libc::EOPNOTSUPP + } +} + +// ibv_qp related inline functions +#[inline] +pub unsafe fn ibv_create_qp_ex( + context: *mut ibv_context, + qp_init_attr_ex: *mut ibv_qp_init_attr_ex, +) -> Option<*mut ibv_qp> { + let mask = ibv_qp_init_attr_mask((*qp_init_attr_ex).comp_mask); + + if mask == ibv_qp_init_attr_mask::IBV_QP_INIT_ATTR_PD { + Some(ibv_create_qp( + (*qp_init_attr_ex).pd, + qp_init_attr_ex as *mut ibv_qp_init_attr, + )) + } else { + let vcr = verbs_get_ctx_op!(context, create_qp_ex); + + if let Some(vctx) = vcr { + Some((*vctx).create_qp_ex.unwrap()(context, qp_init_attr_ex)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } + } +} + +// ibv_td related inline functions +#[inline] +pub unsafe fn ibv_alloc_td( + context: *mut ibv_context, + init_attr: *mut ibv_td_init_attr, +) -> Option<*mut ibv_td> { + let vcr = verbs_get_ctx_op!(context, alloc_td); + + if let Some(vctx) = vcr { + Some((*vctx).alloc_td.unwrap()(context, init_attr)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_dealloc_td(td: *mut ibv_td) -> c_int { + let vcr = verbs_get_ctx_op!((*td).context, dealloc_td); + + if let Some(vctx) = vcr { + (*vctx).dealloc_td.unwrap()(td) + } else { + libc::EOPNOTSUPP + } +} + +// ibv_pd related inline function +#[inline] +pub unsafe fn ibv_alloc_parent_domain( + context: *mut ibv_context, + attr: *mut ibv_parent_domain_init_attr, +) -> Option<*mut ibv_pd> { + let vcr = verbs_get_ctx_op!(context, alloc_parent_domain); + + if let Some(vctx) = vcr { + Some((*vctx).alloc_parent_domain.unwrap()(context, attr)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +// device related inline functions +#[inline] +pub unsafe fn ibv_query_rt_values_ex( + context: *mut ibv_context, + values: *mut ibv_values_ex, +) -> c_int { + let vcr = verbs_get_ctx_op!(context, query_rt_values); + + if let Some(vctx) = vcr { + (*vctx).query_rt_values.unwrap()(context, values) + } else { + libc::EOPNOTSUPP + } +} + +#[inline] +pub unsafe fn ibv_query_device_ex( + context: *mut ibv_context, + input: *const ibv_query_device_ex_input, + attr: *mut ibv_device_attr_ex, +) -> c_int { + let vcr = verbs_get_ctx_op!(context, query_device_ex); + + if let Some(vctx) = vcr { + let ret = (*vctx).query_device_ex.unwrap()(context, input, attr, mem::size_of_val(&*attr)); + if ret != libc::EOPNOTSUPP { + return ret; + } + } + + // TODO: memset(attr, 0, sizeof(*attr)); + ibv_query_device(context, &mut (*attr).orig_attr) +} + +// ibv_qp related inline functions +#[inline] +pub unsafe fn ibv_open_qp( + context: *mut ibv_context, + qp_open_attr: *mut ibv_qp_open_attr, +) -> Option<*mut ibv_qp> { + let vcr = verbs_get_ctx_op!(context, open_qp); + + if let Some(vctx) = vcr { + Some((*vctx).open_qp.unwrap()(context, qp_open_attr)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_modify_qp_rate_limit( + qp: *mut ibv_qp, + attr: *mut ibv_qp_rate_limit_attr, +) -> c_int { + let vcr = verbs_get_ctx_op!((*qp).context, modify_qp_rate_limit); + + if let Some(vctx) = vcr { + (*vctx).modify_qp_rate_limit.unwrap()(qp, attr) + } else { + libc::EOPNOTSUPP + } +} + +// ibv_wq related inline functions +#[inline] +pub unsafe fn ibv_create_wq( + context: *mut ibv_context, + wq_init_attr: *mut ibv_wq_init_attr, +) -> Option<*mut ibv_wq> { + let vcr = verbs_get_ctx_op!(context, create_wq); + + if let Some(vctx) = vcr { + let wq = (*vctx).create_wq.unwrap()(context, wq_init_attr); + if wq != (ptr::null::() as *mut _) { + (*wq).events_completed = 0; + libc::pthread_mutex_init( + &mut (*wq).mutex, + ptr::null::() as *mut _, + ); + libc::pthread_cond_init( + &mut (*wq).cond, + ptr::null::() as *mut _, + ); + } + Some(wq) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_modify_wq(wq: *mut ibv_wq, wq_attr: *mut ibv_wq_attr) -> c_int { + let vcr = verbs_get_ctx_op!((*wq).context, modify_wq); + + if let Some(vctx) = vcr { + (*vctx).modify_wq.unwrap()(wq, wq_attr) + } else { + libc::EOPNOTSUPP + } +} + +#[inline] +pub unsafe fn ibv_destroy_wq(wq: *mut ibv_wq) -> c_int { + let vcr = verbs_get_ctx_op!((*wq).context, destroy_wq); + + if let Some(vctx) = vcr { + (*vctx).destroy_wq.unwrap()(wq) + } else { + libc::EOPNOTSUPP + } +} + +// ibv_rwq_ind_table related inline functions +#[inline] +pub unsafe fn ibv_create_rwq_ind_table( + context: *mut ibv_context, + init_attr: *mut ibv_rwq_ind_table_init_attr, +) -> Option<*mut ibv_rwq_ind_table> { + let vcr = verbs_get_ctx_op!(context, create_rwq_ind_table); + + if let Some(vctx) = vcr { + Some((*vctx).create_rwq_ind_table.unwrap()(context, init_attr)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_destroy_rwq_ind_table(rwq_ind_table: *mut ibv_rwq_ind_table) -> c_int { + let vcr = verbs_get_ctx_op!((*rwq_ind_table).context, destroy_rwq_ind_table); + + if let Some(vctx) = vcr { + (*vctx).destroy_rwq_ind_table.unwrap()(rwq_ind_table) + } else { + libc::EOPNOTSUPP + } +} + +// If IBV_SEND_INLINE flag is set, the data buffers can be reused +// immediately after the call returns. +#[inline] +pub unsafe fn ibv_post_send( + qp: *mut ibv_qp, + wr: *mut ibv_send_wr, + bad_wr: *mut *mut ibv_send_wr, +) -> c_int { + (*(*qp).context).ops.post_send.unwrap()(qp, wr, bad_wr) +} + +#[inline] +pub unsafe fn ibv_post_recv( + qp: *mut ibv_qp, + wr: *mut ibv_recv_wr, + bad_wr: *mut *mut ibv_recv_wr, +) -> c_int { + (*(*qp).context).ops.post_recv.unwrap()(qp, wr, bad_wr) +} + +#[inline] +pub unsafe fn ibv_is_qpt_supported(caps: u32, qpt: ibv_qp_type::Type) -> c_int { + !!(caps & (1 << qpt)) as c_int +} + +// ibv_counters related inline functions +#[inline] +pub unsafe fn ibv_create_counters( + context: *mut ibv_context, + init_attr: *mut ibv_counters_init_attr, +) -> Option<*mut ibv_counters> { + let vcr = verbs_get_ctx_op!(context, create_counters); + + if let Some(vctx) = vcr { + Some((*vctx).create_counters.unwrap()(context, init_attr)) + } else { + *libc::__errno_location() = libc::EOPNOTSUPP; + None + } +} + +#[inline] +pub unsafe fn ibv_destroy_counters(counters: *mut ibv_counters) -> c_int { + let vcr = verbs_get_ctx_op!((*counters).context, destroy_counters); + + if let Some(vctx) = vcr { + (*vctx).destroy_counters.unwrap()(counters) + } else { + libc::EOPNOTSUPP + } +} + +#[inline] +pub unsafe fn ibv_attach_counters_point_flow( + counters: *mut ibv_counters, + attr: *mut ibv_counter_attach_attr, + flow: *mut ibv_flow, +) -> c_int { + let vcr = verbs_get_ctx_op!((*counters).context, attach_counters_point_flow); + + if let Some(vctx) = vcr { + (*vctx).attach_counters_point_flow.unwrap()(counters, attr, flow) + } else { + libc::EOPNOTSUPP + } +} + +#[inline] +pub unsafe fn ibv_read_counters( + counters: *mut ibv_counters, + counters_value: *mut u64, + ncounters: u32, + flags: u32, +) -> c_int { + let vcr = verbs_get_ctx_op!((*counters).context, read_counters); + + if let Some(vctx) = vcr { + (*vctx).read_counters.unwrap()(counters, counters_value, ncounters, flags) + } else { + libc::EOPNOTSUPP + } +} + +/// Inline functions from + +pub const RDMA_IB_IP_PS_MASK: u64 = 0xFFFFFFFFFFFF0000; +pub const RDMA_IB_IP_PORT_MASK: u64 = 0x000000000000FFFF; +pub const RDMA_IB_IP_PS_TCP: u64 = 0x0000000001060000; +pub const RDMA_IB_IP_PS_UDP: u64 = 0x0000000001110000; +pub const RDMA_IB_PS_IB: u64 = 0x00000000013F0000; + +pub const RDMA_UDP_QKEY: u32 = 0x01234567; + +pub const RAI_PASSIVE: u32 = 0x00000001; +pub const RAI_NUMERICHOST: u32 = 0x00000002; +pub const RAI_NOROUTE: u32 = 0x00000004; +pub const RAI_FAMILY: u32 = 0x00000008; + +#[inline] +pub unsafe fn rdma_get_local_addr(id: &rdma_cm_id) -> &libc::sockaddr { + &id.route.addr.src_addr_union.src_addr +} + +#[inline] +pub unsafe fn rdma_get_peer_addr(id: &rdma_cm_id) -> &libc::sockaddr { + &id.route.addr.dst_addr_union.dst_addr +} + +/// Inline functions from + +#[inline] +pub unsafe fn rdma_seterrno(ret: c_int) -> c_int { + if ret != 0 { + *libc::__errno_location() = ret; + -1 + } else { + ret + } +} + +#[inline] +pub unsafe fn rdma_reg_msgs(id: *mut rdma_cm_id, addr: *mut c_void, length: usize) -> *mut ibv_mr { + ibv_reg_mr( + (*id).pd, + addr, + length, + ibv_access_flags::IBV_ACCESS_LOCAL_WRITE.0 as c_int, + ) +} + +#[inline] +pub unsafe fn rdma_reg_read(id: *mut rdma_cm_id, addr: *mut c_void, length: usize) -> *mut ibv_mr { + ibv_reg_mr( + (*id).pd, + addr, + length, + (ibv_access_flags::IBV_ACCESS_LOCAL_WRITE | ibv_access_flags::IBV_ACCESS_REMOTE_READ).0 + as c_int, + ) +} + +#[inline] +pub unsafe fn rdma_reg_write(id: *mut rdma_cm_id, addr: *mut c_void, length: usize) -> *mut ibv_mr { + ibv_reg_mr( + (*id).pd, + addr, + length, + (ibv_access_flags::IBV_ACCESS_LOCAL_WRITE | ibv_access_flags::IBV_ACCESS_REMOTE_WRITE).0 + as c_int, + ) +} + +#[inline] +pub unsafe fn rdma_dereg_mr(mr: *mut ibv_mr) -> c_int { + rdma_seterrno(ibv_dereg_mr(mr)) +} + +#[inline] +pub unsafe fn rdma_post_recvv( + id: *mut rdma_cm_id, + context: *mut c_void, + sgl: *mut ibv_sge, + nsge: c_int, +) -> c_int { + let mut wr = ibv_recv_wr { + wr_id: context as u64, + next: ptr::null::() as *mut _, + sg_list: sgl, + num_sge: nsge, + }; + let mut bad = ptr::null::() as *mut _; + + if (*id).srq as usize != 0 { + rdma_seterrno(ibv_post_srq_recv((*id).srq, &mut wr, &mut bad)) + } else { + rdma_seterrno(ibv_post_recv((*id).qp, &mut wr, &mut bad)) + } +} + +#[inline] +pub unsafe fn rdma_post_sendv( + id: *mut rdma_cm_id, + context: *mut c_void, + sgl: *mut ibv_sge, + nsge: c_int, + flags: c_int, +) -> c_int { + let mut wr = std::mem::zeroed::(); + wr.wr_id = context as u64; + wr.next = ptr::null::() as *mut _; + wr.sg_list = sgl; + wr.num_sge = nsge; + wr.opcode = ibv_wr_opcode::IBV_WR_SEND; + wr.send_flags = flags as c_uint; + let mut bad = ptr::null::() as *mut _; + + rdma_seterrno(ibv_post_send((*id).qp, &mut wr, &mut bad)) +} + +#[inline] +pub unsafe fn rdma_post_readv( + id: *mut rdma_cm_id, + context: *mut c_void, + sgl: *mut ibv_sge, + nsge: c_int, + flags: c_int, + remote_addr: u64, + rkey: u32, +) -> c_int { + let mut wr = std::mem::zeroed::(); + wr.wr_id = context as u64; + wr.next = ptr::null::() as *mut _; + wr.sg_list = sgl; + wr.num_sge = nsge; + wr.opcode = ibv_wr_opcode::IBV_WR_RDMA_READ; + wr.send_flags = flags as c_uint; + wr.wr = wr_t { + rdma: rdma_t { remote_addr, rkey }, + }; + let mut bad = ptr::null::() as *mut _; + + rdma_seterrno(ibv_post_send((*id).qp, &mut wr, &mut bad)) +} + +#[inline] +pub unsafe fn rdma_post_writev( + id: *mut rdma_cm_id, + context: *mut c_void, + sgl: *mut ibv_sge, + nsge: c_int, + flags: c_int, + remote_addr: u64, + rkey: u32, +) -> c_int { + let mut wr = std::mem::zeroed::(); + wr.wr_id = context as u64; + wr.next = ptr::null::() as *mut _; + wr.sg_list = sgl; + wr.num_sge = nsge; + wr.opcode = ibv_wr_opcode::IBV_WR_RDMA_WRITE; + wr.send_flags = flags as c_uint; + wr.wr = wr_t { + rdma: rdma_t { remote_addr, rkey }, + }; + let mut bad = ptr::null::() as *mut _; + + rdma_seterrno(ibv_post_send((*id).qp, &mut wr, &mut bad)) +} + +#[inline] +pub unsafe fn rdma_post_recv( + id: *mut rdma_cm_id, + context: *mut c_void, + addr: *mut c_void, + length: usize, + mr: *mut ibv_mr, +) -> c_int { + assert!( + addr >= (*mr).addr && (addr as usize + length <= (*mr).addr as usize + (*mr).length), + "invalid addr={} and length={}", + addr as usize, + length, + ); + let mut sge = ibv_sge { + addr: addr as u64, + length: length as u32, + lkey: (*mr).lkey, + }; + let nsge = 1; + rdma_post_recvv(id, context, &mut sge, nsge) +} + +#[inline] +pub unsafe fn rdma_post_send( + id: *mut rdma_cm_id, + context: *mut c_void, + addr: *mut c_void, + length: usize, + mr: *mut ibv_mr, + flags: c_int, +) -> c_int { + let mut sge = ibv_sge { + addr: addr as u64, + length: length as u32, + lkey: if !mr.is_null() { (*mr).lkey } else { 0 }, + }; + let nsge = 1; + rdma_post_sendv(id, context, &mut sge, nsge, flags) +} + +#[inline] +pub unsafe fn rdma_post_read( + id: *mut rdma_cm_id, + context: *mut c_void, + addr: *mut c_void, + length: usize, + mr: *mut ibv_mr, + flags: c_int, + remote_addr: u64, + rkey: u32, +) -> c_int { + let mut sge = ibv_sge { + addr: addr as u64, + length: length as u32, + lkey: (*mr).lkey, + }; + let nsge = 1; + rdma_post_readv(id, context, &mut sge, nsge, flags, remote_addr, rkey) +} + +#[inline] +pub unsafe fn rdma_post_write( + id: *mut rdma_cm_id, + context: *mut c_void, + addr: *mut c_void, + length: usize, + mr: *mut ibv_mr, + flags: c_int, + remote_addr: u64, + rkey: u32, +) -> c_int { + let mut sge = ibv_sge { + addr: addr as u64, + length: length as u32, + lkey: if !mr.is_null() { (*mr).lkey } else { 0 }, + }; + let nsge = 1; + rdma_post_writev(id, context, &mut sge, nsge, flags, remote_addr, rkey) +} + +#[inline] +pub unsafe fn rdma_post_ud_send( + id: *mut rdma_cm_id, + context: *mut c_void, + addr: *mut c_void, + length: usize, + mr: *mut ibv_mr, + flags: c_int, + ah: *mut ibv_ah, + remote_qpn: u32, +) -> c_int { + let mut sge = ibv_sge { + addr: addr as u64, + length: length as u32, + lkey: if !mr.is_null() { (*mr).lkey } else { 0 }, + }; + + let mut wr = std::mem::zeroed::(); + wr.wr_id = context as u64; + wr.next = ptr::null::() as *mut _; + wr.sg_list = &mut sge; + wr.num_sge = 1; + wr.opcode = ibv_wr_opcode::IBV_WR_SEND; + wr.send_flags = flags as c_uint; + wr.wr = wr_t { + ud: ud_t { + ah, + remote_qpn, + remote_qkey: RDMA_UDP_QKEY, + }, + }; + let mut bad = ptr::null::() as *mut _; + + rdma_seterrno(ibv_post_send((*id).qp, &mut wr, &mut bad)) +} + +#[inline] +pub unsafe fn rdma_get_send_comp(id: *mut rdma_cm_id, wc: *mut ibv_wc) -> c_int { + let mut ret: c_int; + let mut cq = ptr::null::() as *mut _; + let mut context = ptr::null::() as *mut _; + let nevents = 1; + let num_entries = 1; + let solicited_only = 0; + + loop { + ret = ibv_poll_cq((*id).send_cq, num_entries, wc); + if ret != 0 { + break; + } + ret = ibv_req_notify_cq((*id).send_cq, solicited_only); + if ret != 0 { + return rdma_seterrno(ret); + } + ret = ibv_poll_cq((*id).send_cq, num_entries, wc); + if ret != 0 { + break; + } + ret = ibv_get_cq_event((*id).send_cq_channel, &mut cq, &mut context); + if ret != 0 { + return ret; + } + + assert!(cq == (*id).send_cq && context as *mut rdma_cm_id == id); + ibv_ack_cq_events((*id).send_cq, nevents); + } + if ret < 0 { + rdma_seterrno(ret) + } else { + ret + } +} + +#[inline] +pub unsafe fn rdma_get_recv_comp(id: *mut rdma_cm_id, wc: *mut ibv_wc) -> c_int { + let mut ret: c_int; + let mut cq = ptr::null::() as *mut _; + let mut context = ptr::null::() as *mut _; + let nevents = 1; + let num_entries = 1; + let solicited_only = 0; + loop { + ret = ibv_poll_cq((*id).recv_cq, num_entries, wc); + if ret != 0 { + break; + } + ret = ibv_req_notify_cq((*id).recv_cq, solicited_only); + if ret != 0 { + return rdma_seterrno(ret); + } + ret = ibv_poll_cq((*id).recv_cq, num_entries, wc); + if ret != 0 { + break; + } + ret = ibv_get_cq_event((*id).recv_cq_channel, &mut cq, &mut context); + if ret != 0 { + return ret; + } + + assert!(cq == (*id).recv_cq && context as *mut rdma_cm_id == id); + ibv_ack_cq_events((*id).recv_cq, nevents); + } + if ret < 0 { + rdma_seterrno(ret) + } else { + ret + } +}