mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-07-19 22:42:20 +00:00
rdma-engine: standalone cross-process RDMA-READ target + client
Split the in-process loopback proof into separate target/client
binaries so the {remote_addr, rkey} addressing genuinely crosses a
process boundary (the precursor to an engine<->volume-server split):
- rdma-target <bind_ip> <port> <file>: loads a file, registers it for
REMOTE_READ, serves one client over rxe.
- rdma-client <ip> <port> <out>: connects, RDMA-READs the whole buffer
into a fresh local buffer, writes it out.
Verified over SoftRoCE (rxe0): an 8 MiB real file served by the target
process is RDMA-read by a separate client process (status=Success,
8388608 bytes) and md5 matches the source. Real cross-process RDMA of
real data; serving from a needle-backed buffer in the volume server is
the next (multi-week) step.
This commit is contained in:
@@ -17,6 +17,17 @@ name = "rdma-loopback"
|
||||
path = "src/bin/rdma_loopback.rs"
|
||||
required-features = ["real-rdma"]
|
||||
|
||||
# Standalone cross-process RDMA target + client (real file data over rxe).
|
||||
[[bin]]
|
||||
name = "rdma-target"
|
||||
path = "src/bin/rdma_target.rs"
|
||||
required-features = ["real-rdma"]
|
||||
|
||||
[[bin]]
|
||||
name = "rdma-client"
|
||||
path = "src/bin/rdma_client.rs"
|
||||
required-features = ["real-rdma"]
|
||||
|
||||
[lib]
|
||||
name = "rdma_engine"
|
||||
path = "src/lib.rs"
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
//! Standalone RDMA-READ client: connect to an `rdma-target`, RDMA-READ its whole
|
||||
//! registered buffer into a fresh local buffer (separate process / address
|
||||
//! space), and write it out. md5 of the output vs the target's source file
|
||||
//! proves a real cross-process RDMA read of real data.
|
||||
//!
|
||||
//! cargo run --features real-rdma --bin rdma-client -- <target_ip> <port> <out_file>
|
||||
|
||||
use std::fs;
|
||||
|
||||
use rdma_engine::rdma_real::RealRdmaContext;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() < 4 {
|
||||
eprintln!("usage: rdma-client <target_ip> <port> <out_file>");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let ip = args[1].clone();
|
||||
let port: u16 = args[2].parse().expect("port");
|
||||
let out = &args[3];
|
||||
|
||||
let (ctx, raddr, rkey, rlen) = match RealRdmaContext::connect(&ip, port) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
eprintln!("client connect failed: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
eprintln!("client: connected to {ip}:{port}, remote MR len={rlen} rkey=0x{rkey:x}");
|
||||
|
||||
let mut dest = vec![0u8; rlen];
|
||||
let (_mi, mr) = ctx.register_memory(&mut dest).expect("register dest");
|
||||
ctx.post_read(&mut dest, mr, raddr, rkey, 1).expect("post_read");
|
||||
let wc = ctx.poll_completion().expect("poll completion");
|
||||
ctx.finish();
|
||||
eprintln!(
|
||||
"client: RDMA READ status={:?} byte_len={}",
|
||||
wc.status, wc.byte_len
|
||||
);
|
||||
|
||||
fs::write(out, &dest).expect("write output");
|
||||
eprintln!("client: wrote {} bytes to {out}", dest.len());
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
//! Standalone RDMA-READ target: load a file, register it for `REMOTE_READ`, and
|
||||
//! serve one client over SoftRoCE (rxe) — a real, separate-process precursor to
|
||||
//! a volume-side RDMA target. Pairs with `rdma-client`.
|
||||
//!
|
||||
//! cargo run --features real-rdma --bin rdma-target -- <bind_ip> <port> <file>
|
||||
//!
|
||||
//! `<bind_ip>` must be an IP on the rxe-backing netdev (e.g. eth0's IP).
|
||||
|
||||
use std::fs;
|
||||
|
||||
use rdma_engine::rdma_real::RealResponder;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() < 4 {
|
||||
eprintln!("usage: rdma-target <bind_ip> <port> <file>");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let ip = args[1].clone();
|
||||
let port: u16 = args[2].parse().expect("port");
|
||||
let file = &args[3];
|
||||
|
||||
let mut data = fs::read(file).expect("read source file");
|
||||
eprintln!(
|
||||
"target: serving {} bytes of {file} on {ip}:{port} (REMOTE_READ)",
|
||||
data.len()
|
||||
);
|
||||
|
||||
match RealResponder::serve_once(&ip, port, &mut data, || {
|
||||
eprintln!("target: listening; waiting for a client");
|
||||
}) {
|
||||
Ok(()) => eprintln!("target: served one client; done"),
|
||||
Err(e) => {
|
||||
eprintln!("target error: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user