rdma-engine: standalone cross-process RDMA-READ target + client

Split the in-process loopback proof into separate target/client
binaries so the {remote_addr, rkey} addressing genuinely crosses a
process boundary (the precursor to an engine<->volume-server split):

- rdma-target <bind_ip> <port> <file>: loads a file, registers it for
  REMOTE_READ, serves one client over rxe.
- rdma-client <ip> <port> <out>: connects, RDMA-READs the whole buffer
  into a fresh local buffer, writes it out.

Verified over SoftRoCE (rxe0): an 8 MiB real file served by the target
process is RDMA-read by a separate client process (status=Success,
8388608 bytes) and md5 matches the source. Real cross-process RDMA of
real data; serving from a needle-backed buffer in the volume server is
the next (multi-week) step.
This commit is contained in:
Chris Lu
2026-06-14 18:50:06 -07:00
parent 96722ebf4b
commit 49c8d123b5
3 changed files with 92 additions and 0 deletions
@@ -17,6 +17,17 @@ name = "rdma-loopback"
path = "src/bin/rdma_loopback.rs"
required-features = ["real-rdma"]
# Standalone cross-process RDMA target + client (real file data over rxe).
[[bin]]
name = "rdma-target"
path = "src/bin/rdma_target.rs"
required-features = ["real-rdma"]
[[bin]]
name = "rdma-client"
path = "src/bin/rdma_client.rs"
required-features = ["real-rdma"]
[lib]
name = "rdma_engine"
path = "src/lib.rs"
@@ -0,0 +1,43 @@
//! Standalone RDMA-READ client: connect to an `rdma-target`, RDMA-READ its whole
//! registered buffer into a fresh local buffer (separate process / address
//! space), and write it out. md5 of the output vs the target's source file
//! proves a real cross-process RDMA read of real data.
//!
//! cargo run --features real-rdma --bin rdma-client -- <target_ip> <port> <out_file>
use std::fs;
use rdma_engine::rdma_real::RealRdmaContext;
fn main() {
let args: Vec<String> = std::env::args().collect();
if args.len() < 4 {
eprintln!("usage: rdma-client <target_ip> <port> <out_file>");
std::process::exit(2);
}
let ip = args[1].clone();
let port: u16 = args[2].parse().expect("port");
let out = &args[3];
let (ctx, raddr, rkey, rlen) = match RealRdmaContext::connect(&ip, port) {
Ok(v) => v,
Err(e) => {
eprintln!("client connect failed: {e}");
std::process::exit(1);
}
};
eprintln!("client: connected to {ip}:{port}, remote MR len={rlen} rkey=0x{rkey:x}");
let mut dest = vec![0u8; rlen];
let (_mi, mr) = ctx.register_memory(&mut dest).expect("register dest");
ctx.post_read(&mut dest, mr, raddr, rkey, 1).expect("post_read");
let wc = ctx.poll_completion().expect("poll completion");
ctx.finish();
eprintln!(
"client: RDMA READ status={:?} byte_len={}",
wc.status, wc.byte_len
);
fs::write(out, &dest).expect("write output");
eprintln!("client: wrote {} bytes to {out}", dest.len());
}
@@ -0,0 +1,38 @@
//! Standalone RDMA-READ target: load a file, register it for `REMOTE_READ`, and
//! serve one client over SoftRoCE (rxe) — a real, separate-process precursor to
//! a volume-side RDMA target. Pairs with `rdma-client`.
//!
//! cargo run --features real-rdma --bin rdma-target -- <bind_ip> <port> <file>
//!
//! `<bind_ip>` must be an IP on the rxe-backing netdev (e.g. eth0's IP).
use std::fs;
use rdma_engine::rdma_real::RealResponder;
fn main() {
let args: Vec<String> = std::env::args().collect();
if args.len() < 4 {
eprintln!("usage: rdma-target <bind_ip> <port> <file>");
std::process::exit(2);
}
let ip = args[1].clone();
let port: u16 = args[2].parse().expect("port");
let file = &args[3];
let mut data = fs::read(file).expect("read source file");
eprintln!(
"target: serving {} bytes of {file} on {ip}:{port} (REMOTE_READ)",
data.len()
);
match RealResponder::serve_once(&ip, port, &mut data, || {
eprintln!("target: listening; waiting for a client");
}) {
Ok(()) => eprintln!("target: served one client; done"),
Err(e) => {
eprintln!("target error: {e}");
std::process::exit(1);
}
}
}