diff --git a/net.hh b/net.hh index 1befaf61da..ce98bafa28 100644 --- a/net.hh +++ b/net.hh @@ -9,25 +9,94 @@ namespace net { +class packet; + struct fragment { char* base; size_t size; }; -struct packet { - ~packet() { - if (len) { - completed.set_value(); - } - } - packet() = default; - packet(packet&& x) - : fragments(std::move(x.fragments)), len(x.len), completed(std::move(x.completed)) { - x.len = 0; +// Zero-copy friendly packet class +// +// For implementing zero-copy, we need a flexible destructor that can +// destroy packet data in different ways: decrementing a reference count, +// or calling a free()-like function. +// +// Moreover, we need different destructors for each set of fragments within +// a single fragment. For example, a header and trailer might need delete[] +// to be called, while the internal data needs a reference count to be +// released. Matters are complicated in that fragments can be split +// (due to virtual/physical translation). +// +// To implement this, we associate each packet with a single destructor, +// but allow composing a packet from another packet plus a fragment to +// be added, with its own destructor, causing the destructors to be chained. +// +// The downside is that the data needed for the destructor is duplicated, +// if it is already available in the fragment itself. +// +// As an optimization, when we allocate small fragments, we allocate some +// extra space, so prepending to the packet does not require extra +// allocations. This is useful when adding headers. +// +class packet final { + // enough for lots of headers, not quite two cache lines: + static constexpr size_t internal_data_size = 128 - 16; + struct deleter { + std::unique_ptr next; + deleter(std::unique_ptr next) : next(std::move(next)) {} + virtual ~deleter() {}; + }; + struct internal_deleter final : deleter { + // FIXME: make buf an std::array<>? + char* buf; + unsigned free_head; + internal_deleter(std::unique_ptr next, char* buf, unsigned free_head) + : deleter(std::move(next)), buf(buf), free_head(free_head) {} + virtual ~internal_deleter() override { delete[] buf; } + }; + template + struct lambda_deleter final : deleter { + Deleter del; + lambda_deleter(std::unique_ptr next, Deleter&& del) + : deleter(std::move(next)), del(std::move(del)) {} + virtual ~lambda_deleter() override { del(); } + }; + template + std::unique_ptr + make_deleter(std::unique_ptr next, Deleter d) { + return std::unique_ptr(new lambda_deleter(std::move(next), std::move(d))); } + // when destroyed, virtual destructor will reclaim resources + std::unique_ptr _deleter; +public: std::vector fragments; unsigned len = 0; - promise<> completed; +public: + // build empty packet + packet() = default; + // move existing packet + packet(packet&& x); + // copy data into packet + packet(char* data, size_t len); + // copy data into packet + packet(fragment frag); + // zero-copy single fragment + template + packet(fragment frag, Deleter deleter); + // zero-copy multiple fragment + template + packet(std::vector frag, Deleter deleter); + // append fragment (copying new fragment) + packet(packet&& x, fragment frag); + // prepend fragment (copying new fragment, with header optimization) + packet(fragment frag, packet&& x); + // prepend fragment (zero-copy) + template + packet(fragment frag, Deleter deleter, packet&& x); + // append fragment (zero-copy) + template + packet(packet&& x, fragment frag, Deleter deleter); }; class device { @@ -37,6 +106,100 @@ public: virtual future<> send(packet p) = 0; }; +inline +packet::packet(packet&& x) + : _deleter(std::move(x._deleter)), fragments(std::move(x.fragments)), len(x.len) { + x.len = 0; +} + +inline +packet::packet(fragment frag) : len(frag.size) { + auto flen = std::max(frag.size, internal_data_size); + std::unique_ptr buf{new char[flen]}; + std::copy(frag.base, frag.base + frag.size, buf.get() + flen - frag.size); + fragments.push_back(frag); + _deleter.reset(new internal_deleter(std::unique_ptr(), + buf.release(), flen - frag.size)); +} + +inline +packet::packet(char* data, size_t size) : packet(fragment{data, size}) { +} + +template +inline +packet::packet(fragment frag, Deleter d) + : _deleter(make_deleter(std::unique_ptr(), std::move(d))) { + fragments.push_back(frag); + len = frag.size; +} + +template +inline +packet::packet(std::vector frag, Deleter d) + : _deleter(make_deleter(std::unique_ptr(), std::move(d))) + , fragments(std::move(frag)) + , len(0) { + for (auto&& f : fragments) { + len += f.size; + } +} + +inline +packet::packet(packet&& x, fragment frag) + : fragments(std::move(x.fragments)) + , len(x.len + frag.size) { + std::unique_ptr buf(new char[frag.size]); + std::copy(frag.base, frag.base + frag.size, buf.get()); + _deleter = make_deleter(std::move(x._deleter), [buf = buf.release()] { + delete[] buf; + }); + x.len = 0; +} + +inline +packet::packet(fragment frag, packet&& x) + : _deleter(std::move(x._deleter)), len(x.len + frag.size) { + // try to prepend into existing internal fragment + auto id = dynamic_cast(x._deleter.get()); + if (id && id->free_head >= frag.size) { + id->free_head -= frag.size; + fragments[0].base -= frag.size; + fragments[0].size += frag.size; + std::copy(frag.base, frag.base + frag.size, fragments[0].base); + } else { + // didn't work out, allocate and copy + auto size = std::max(frag.size, internal_data_size); + std::unique_ptr buf(new char[size]); + fragments.reserve(x.fragments.size() + 1); + fragments.push_back({buf.get() + size - frag.size, frag.size}); + std::copy(x.fragments.begin(), x.fragments.end(), std::back_inserter(fragments)); + x.fragments.clear(); + _deleter.reset(new internal_deleter(std::move(_deleter), buf.release(), size - frag.size)); + } + x.len = 0; +} + +template +inline +packet::packet(fragment frag, Deleter d, packet&& x) : len(x.len + frag.size) { + fragments.reserve(x.fragments.size() + 1); + fragments.push_back(frag); + std::copy(x.fragments.begin(), x.fragments.end(), std::back_inserter(fragments)); + x.fragments.clear(); + _deleter.reset(make_deleter(std::move(_deleter), d)); + x.len = 0; +} + +template +inline +packet::packet(packet&& x, fragment frag, Deleter d) + : fragments(std::move(x.fragments)), len(x.len + frag.size) { + fragments.push_back(frag); + _deleter.reset(make_deleter(std::move(_deleter), d)); + x.len = 0; +} + } #endif /* NET_HH_ */ diff --git a/virtio.cc b/virtio.cc index 94ad763169..fa6bf75fda 100644 --- a/virtio.cc +++ b/virtio.cc @@ -382,11 +382,8 @@ virtio_net_device::rxq::prepare_buffers(semaphore& available) { b.len = 4096; b.writeable = true; b.completed.get_future().then([this, buf = buf.get()] (size_t len) { - packet p; - p.fragments.push_back(fragment{buf + _header_len, len - _header_len}); - p.completed.get_future().then([buf] { - delete[] buf; - }); + packet p(fragment{buf + _header_len, len - _header_len}, + [buf] { delete[] buf; }); _dev.queue_rx_packet(std::move(p)); }); bc.push_back(std::move(b));