From a7930ffcaa9fb39a441d2bb2674d5ba0a6749da7 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 28 Aug 2014 17:29:27 +0300 Subject: [PATCH] net: rework packet class 1. Replace the completion promise<> with a custom deleter class; this is lighter weight, and we don't really need the destructor to be executed by the scheduler. 2. Add lots of consuctors for composing packets from existing packet, by appending or prepending packets 3. Over-allocate in some cases to accomodate for the common practice of prepending protocol headers. --- net.hh | 185 ++++++++++++++++++++++++++++++++++++++++++++++++++---- virtio.cc | 7 +-- 2 files changed, 176 insertions(+), 16 deletions(-) diff --git a/net.hh b/net.hh index 1befaf61da..ce98bafa28 100644 --- a/net.hh +++ b/net.hh @@ -9,25 +9,94 @@ namespace net { +class packet; + struct fragment { char* base; size_t size; }; -struct packet { - ~packet() { - if (len) { - completed.set_value(); - } - } - packet() = default; - packet(packet&& x) - : fragments(std::move(x.fragments)), len(x.len), completed(std::move(x.completed)) { - x.len = 0; +// Zero-copy friendly packet class +// +// For implementing zero-copy, we need a flexible destructor that can +// destroy packet data in different ways: decrementing a reference count, +// or calling a free()-like function. +// +// Moreover, we need different destructors for each set of fragments within +// a single fragment. For example, a header and trailer might need delete[] +// to be called, while the internal data needs a reference count to be +// released. Matters are complicated in that fragments can be split +// (due to virtual/physical translation). +// +// To implement this, we associate each packet with a single destructor, +// but allow composing a packet from another packet plus a fragment to +// be added, with its own destructor, causing the destructors to be chained. +// +// The downside is that the data needed for the destructor is duplicated, +// if it is already available in the fragment itself. +// +// As an optimization, when we allocate small fragments, we allocate some +// extra space, so prepending to the packet does not require extra +// allocations. This is useful when adding headers. +// +class packet final { + // enough for lots of headers, not quite two cache lines: + static constexpr size_t internal_data_size = 128 - 16; + struct deleter { + std::unique_ptr next; + deleter(std::unique_ptr next) : next(std::move(next)) {} + virtual ~deleter() {}; + }; + struct internal_deleter final : deleter { + // FIXME: make buf an std::array<>? + char* buf; + unsigned free_head; + internal_deleter(std::unique_ptr next, char* buf, unsigned free_head) + : deleter(std::move(next)), buf(buf), free_head(free_head) {} + virtual ~internal_deleter() override { delete[] buf; } + }; + template + struct lambda_deleter final : deleter { + Deleter del; + lambda_deleter(std::unique_ptr next, Deleter&& del) + : deleter(std::move(next)), del(std::move(del)) {} + virtual ~lambda_deleter() override { del(); } + }; + template + std::unique_ptr + make_deleter(std::unique_ptr next, Deleter d) { + return std::unique_ptr(new lambda_deleter(std::move(next), std::move(d))); } + // when destroyed, virtual destructor will reclaim resources + std::unique_ptr _deleter; +public: std::vector fragments; unsigned len = 0; - promise<> completed; +public: + // build empty packet + packet() = default; + // move existing packet + packet(packet&& x); + // copy data into packet + packet(char* data, size_t len); + // copy data into packet + packet(fragment frag); + // zero-copy single fragment + template + packet(fragment frag, Deleter deleter); + // zero-copy multiple fragment + template + packet(std::vector frag, Deleter deleter); + // append fragment (copying new fragment) + packet(packet&& x, fragment frag); + // prepend fragment (copying new fragment, with header optimization) + packet(fragment frag, packet&& x); + // prepend fragment (zero-copy) + template + packet(fragment frag, Deleter deleter, packet&& x); + // append fragment (zero-copy) + template + packet(packet&& x, fragment frag, Deleter deleter); }; class device { @@ -37,6 +106,100 @@ public: virtual future<> send(packet p) = 0; }; +inline +packet::packet(packet&& x) + : _deleter(std::move(x._deleter)), fragments(std::move(x.fragments)), len(x.len) { + x.len = 0; +} + +inline +packet::packet(fragment frag) : len(frag.size) { + auto flen = std::max(frag.size, internal_data_size); + std::unique_ptr buf{new char[flen]}; + std::copy(frag.base, frag.base + frag.size, buf.get() + flen - frag.size); + fragments.push_back(frag); + _deleter.reset(new internal_deleter(std::unique_ptr(), + buf.release(), flen - frag.size)); +} + +inline +packet::packet(char* data, size_t size) : packet(fragment{data, size}) { +} + +template +inline +packet::packet(fragment frag, Deleter d) + : _deleter(make_deleter(std::unique_ptr(), std::move(d))) { + fragments.push_back(frag); + len = frag.size; +} + +template +inline +packet::packet(std::vector frag, Deleter d) + : _deleter(make_deleter(std::unique_ptr(), std::move(d))) + , fragments(std::move(frag)) + , len(0) { + for (auto&& f : fragments) { + len += f.size; + } +} + +inline +packet::packet(packet&& x, fragment frag) + : fragments(std::move(x.fragments)) + , len(x.len + frag.size) { + std::unique_ptr buf(new char[frag.size]); + std::copy(frag.base, frag.base + frag.size, buf.get()); + _deleter = make_deleter(std::move(x._deleter), [buf = buf.release()] { + delete[] buf; + }); + x.len = 0; +} + +inline +packet::packet(fragment frag, packet&& x) + : _deleter(std::move(x._deleter)), len(x.len + frag.size) { + // try to prepend into existing internal fragment + auto id = dynamic_cast(x._deleter.get()); + if (id && id->free_head >= frag.size) { + id->free_head -= frag.size; + fragments[0].base -= frag.size; + fragments[0].size += frag.size; + std::copy(frag.base, frag.base + frag.size, fragments[0].base); + } else { + // didn't work out, allocate and copy + auto size = std::max(frag.size, internal_data_size); + std::unique_ptr buf(new char[size]); + fragments.reserve(x.fragments.size() + 1); + fragments.push_back({buf.get() + size - frag.size, frag.size}); + std::copy(x.fragments.begin(), x.fragments.end(), std::back_inserter(fragments)); + x.fragments.clear(); + _deleter.reset(new internal_deleter(std::move(_deleter), buf.release(), size - frag.size)); + } + x.len = 0; +} + +template +inline +packet::packet(fragment frag, Deleter d, packet&& x) : len(x.len + frag.size) { + fragments.reserve(x.fragments.size() + 1); + fragments.push_back(frag); + std::copy(x.fragments.begin(), x.fragments.end(), std::back_inserter(fragments)); + x.fragments.clear(); + _deleter.reset(make_deleter(std::move(_deleter), d)); + x.len = 0; +} + +template +inline +packet::packet(packet&& x, fragment frag, Deleter d) + : fragments(std::move(x.fragments)), len(x.len + frag.size) { + fragments.push_back(frag); + _deleter.reset(make_deleter(std::move(_deleter), d)); + x.len = 0; +} + } #endif /* NET_HH_ */ diff --git a/virtio.cc b/virtio.cc index 94ad763169..fa6bf75fda 100644 --- a/virtio.cc +++ b/virtio.cc @@ -382,11 +382,8 @@ virtio_net_device::rxq::prepare_buffers(semaphore& available) { b.len = 4096; b.writeable = true; b.completed.get_future().then([this, buf = buf.get()] (size_t len) { - packet p; - p.fragments.push_back(fragment{buf + _header_len, len - _header_len}); - p.completed.get_future().then([buf] { - delete[] buf; - }); + packet p(fragment{buf + _header_len, len - _header_len}, + [buf] { delete[] buf; }); _dev.queue_rx_packet(std::move(p)); }); bc.push_back(std::move(b));