diff --git a/core/xen/evtchn.cc b/core/xen/evtchn.cc index 149c23f1a2..4f5e0eb42f 100644 --- a/core/xen/evtchn.cc +++ b/core/xen/evtchn.cc @@ -85,7 +85,6 @@ public: }; void kernel_evtchn::make_ready(void *arg) { - printf("Got an interrupt!\n"); int fd = reinterpret_cast(arg); uint64_t one = 1; ::write(fd, &one, sizeof(one)); diff --git a/core/xen/gntalloc.cc b/core/xen/gntalloc.cc index 2c43f57743..d835b4577c 100644 --- a/core/xen/gntalloc.cc +++ b/core/xen/gntalloc.cc @@ -8,15 +8,19 @@ #include "osv_xen.hh" #include "gntalloc.hh" +gntref invalid_ref; + // FIXME: Most of the destructors are yet to be coded // class userspace_grant_head : public grant_head { std::atomic _ref_head = { 0 }; + std::vector _refs; public: - userspace_grant_head(std::vector v) : grant_head(v) {} - virtual gntref& new_ref() override; - virtual gntref& new_ref(void *addr, size_t size) override; + userspace_grant_head(std::vector v) : _refs(v) {} + virtual gntref new_ref() override; + virtual gntref new_ref(void *addr, size_t size) override; + virtual void free_ref(gntref& ref); }; class userspace_gntalloc : public gntalloc { @@ -27,7 +31,7 @@ public: explicit userspace_gntalloc(unsigned otherend); ~userspace_gntalloc(); virtual gntref alloc_ref() override; - virtual grant_head *alloc_ref(unsigned refs, bool alloc) override; + virtual grant_head *alloc_ref(unsigned refs) override; }; userspace_gntalloc::userspace_gntalloc(unsigned otherend) @@ -50,7 +54,7 @@ userspace_gntalloc::get_gref(unsigned nr_ents) return gref; } -grant_head *userspace_gntalloc::alloc_ref(unsigned nr_ents, bool alloc) { +grant_head *userspace_gntalloc::alloc_ref(unsigned nr_ents) { auto gref = get_gref(nr_ents); @@ -77,24 +81,30 @@ gntref userspace_gntalloc::alloc_ref() { return p; } -gntref& userspace_grant_head::new_ref() { +gntref userspace_grant_head::new_ref() { return _refs[_id++ % _refs.size()]; } -gntref& userspace_grant_head::new_ref(void *addr, size_t size) { +gntref userspace_grant_head::new_ref(void *addr, size_t size) { gntref& ref = _refs[_id % _refs.size()]; memcpy(ref.page, addr, size); return ref; } +void userspace_grant_head::free_ref(gntref& ref) { + abort(); +} + #ifdef HAVE_OSV class kernel_gntalloc; class kernel_grant_head : public grant_head { + uint32_t _head; public: - kernel_grant_head(std::vector r) : grant_head(r) {} - virtual gntref& new_ref() override; - virtual gntref& new_ref(void *addr, size_t size) override; + kernel_grant_head(uint32_t head) : _head(head) {} + virtual gntref new_ref() override; + virtual gntref new_ref(void *addr, size_t size) override; + virtual void free_ref(gntref& ref); }; class kernel_gntalloc : public gntalloc { @@ -108,7 +118,7 @@ public: kernel_gntalloc(unsigned otherend) : gntalloc(otherend) {} virtual gntref alloc_ref() override; - virtual grant_head *alloc_ref(unsigned refs, bool alloc) override; + virtual grant_head *alloc_ref(unsigned refs) override; friend class kernel_grant_head; }; @@ -119,25 +129,28 @@ virt_to_mfn(void *virt) { return virt_to_phys(virt) >> 12; } -gntref& kernel_grant_head::new_ref() { - return _refs[_id++ % _refs.size()]; +gntref kernel_grant_head::new_ref() { + auto gnt = dynamic_cast(gntalloc::instance()); + + auto ref = gnttab_claim_grant_reference(&_head); + auto page = gnt->new_frame(); + gnttab_grant_foreign_access_ref(ref, gnt->_otherend, virt_to_mfn(page), 0); + return gntref(ref, page); } -gntref& kernel_grant_head::new_ref(void *addr, size_t size) { - - gntref& ref = _refs[_id % _refs.size()]; +gntref kernel_grant_head::new_ref(void *addr, size_t size) { auto gnt = dynamic_cast(gntalloc::instance()); // FIXME: if we can guarantee that the packet allocation came from malloc, not // mmap, we can grant it directly, without copying. We would also have to propagate // the offset information, but that is easier - ref.page = gnt->new_frame(); - memcpy(ref.page, addr, size); + auto page = gnt->new_frame(); + memcpy(page, addr, size); - gnttab_grant_foreign_access_ref(ref.xen_id, gnt->_otherend, virt_to_mfn(ref.page), 0); - - return _refs[_id++ % _refs.size()]; + auto ref = gnttab_claim_grant_reference(&_head); + gnttab_grant_foreign_access_ref(ref, gnt->_otherend, virt_to_mfn(page), 0); + return gntref(ref, page); } void *kernel_gntalloc::new_frame() { @@ -153,10 +166,10 @@ gntref kernel_gntalloc::alloc_ref() { throw std::runtime_error("Failed to initialize allocate grant\n"); } - return {int(ref), page}; + return gntref(int(ref), page); } -grant_head *kernel_gntalloc::alloc_ref(unsigned nr_ents, bool alloc) { +grant_head *kernel_gntalloc::alloc_ref(unsigned nr_ents) { std::vector v; uint32_t head; @@ -164,17 +177,15 @@ grant_head *kernel_gntalloc::alloc_ref(unsigned nr_ents, bool alloc) { if (gnttab_alloc_grant_references(nr_ents, &head)) { throw std::runtime_error("Failed to initialize allocate grant\n"); } - for (unsigned i = 0; i < nr_ents; ++i) { - auto ref = gnttab_claim_grant_reference(&head); - void *page = nullptr; - if (alloc) { - page = new_frame(); - gnttab_grant_foreign_access_ref(ref, _otherend, virt_to_mfn(page), 0); - } - v.push_back({ref, page}); - } - return new kernel_grant_head(v); + return new kernel_grant_head(head); +} + +void kernel_grant_head::free_ref(gntref& ref) { + gnttab_end_foreign_access_ref(ref.xen_id); + gnttab_release_grant_reference(&_head, ref.xen_id); + free(ref.page); + ref = invalid_ref; } #endif diff --git a/core/xen/gntalloc.hh b/core/xen/gntalloc.hh index 398642c257..fde8462aad 100644 --- a/core/xen/gntalloc.hh +++ b/core/xen/gntalloc.hh @@ -3,9 +3,15 @@ #include "core/posix.hh" -struct gntref { +class gntref { +public: int xen_id; void* page; + bool operator==(const gntref &a) { return (xen_id == a.xen_id) && (page == a.page); } + gntref& operator=(const gntref &a) { xen_id = a.xen_id; page = a.page; return *this; } + gntref(int id, void *page) : xen_id(id), page(page) {} + gntref() : xen_id(-1), page(nullptr) {} + operator bool() const { return xen_id != -1 && page != nullptr; } }; class gntalloc; @@ -13,11 +19,10 @@ class gntalloc; class grant_head { protected: unsigned _id = 0; - std::vector _refs; public: - grant_head(std::vector r) : _refs(r) {} - virtual gntref& new_ref() = 0; - virtual gntref& new_ref(void *addr, size_t size) = 0; + virtual gntref new_ref() = 0; + virtual gntref new_ref(void *addr, size_t size) = 0; + virtual void free_ref(gntref& ref) = 0; }; class gntalloc { @@ -32,7 +37,9 @@ public: // The kernel interface can defer allocation, userspace allocation // cannot. The boolean "alloc" tell us whether or not we should allocate // now or try to defer. - virtual grant_head *alloc_ref(unsigned nr_ents, bool alloc) = 0; + virtual grant_head *alloc_ref(unsigned nr_ents) = 0; friend class grant_head; }; + +extern gntref invalid_ref; #endif diff --git a/core/xen/osv_xen.hh b/core/xen/osv_xen.hh index 49630e76a8..08159d9512 100644 --- a/core/xen/osv_xen.hh +++ b/core/xen/osv_xen.hh @@ -24,6 +24,10 @@ gnttab_claim_grant_reference(uint32_t *private_head); extern int gnttab_grant_foreign_access(uint16_t domid, unsigned long frame, int readonly, uint32_t *result); extern void gnttab_grant_foreign_access_ref(uint32_t ref, uint16_t domid, unsigned long frame, int readonly); +extern void +gnttab_release_grant_reference(uint32_t *private_head, unsigned ref); +extern int +gnttab_end_foreign_access_ref(unsigned ref); extern "C" uint64_t virt_to_phys(void *virt); diff --git a/net/xenfront.cc b/net/xenfront.cc index b77467c002..a74cede9e3 100644 --- a/net/xenfront.cc +++ b/net/xenfront.cc @@ -63,8 +63,10 @@ private: int bind_rx_evtchn(); future<> alloc_rx_references(unsigned refs); + future<> handle_tx_completions(); future<> queue_rx_packet(); + void alloc_one_rx_reference(unsigned id); std::string path(std::string s) { return _device_str + "/" + s; } public: @@ -111,7 +113,7 @@ xenfront_net_device::send(packet _p) { // FIXME: negotiate and use scatter/gather _p.linearize(); - return _tx_ring.free_idx().then([this, p = std::move(_p), frag] (uint32_t idx) mutable { + return _tx_ring.entries.get_index().then([this, p = std::move(_p), frag] (unsigned idx) mutable { auto req_prod = _tx_ring._sring->req_prod; @@ -119,6 +121,8 @@ xenfront_net_device::send(packet _p) { auto ref = _tx_refs->new_ref(f.base, f.size); + assert(!_tx_ring.entries[idx]); + _tx_ring.entries[idx] = ref; auto req = &_tx_ring._sring->_ring[idx].req; @@ -139,7 +143,6 @@ xenfront_net_device::send(packet _p) { _tx_ring._sring->req_event++; if ((frag + 1) == p.nr_frags()) { - printf("NOTIFY!!\n"); _evtchn->notify(_tx_evtchn); return make_ready_future<>(); } else { @@ -153,16 +156,18 @@ xenfront_net_device::send(packet _p) { #define rmb() asm volatile("lfence":::"memory"); #define wmb() asm volatile("":::"memory"); -// FIXME: This is totally wrong, just coded so we can gt started with sending template -future front_ring::free_idx() { - static uint32_t idx = 0; - return make_ready_future(idx++); +future front_ring::entries::get_index() { + return _ids.pop_eventually(); +} + +template +future<> front_ring::entries::free_index(unsigned id) { + return _ids.push_eventually(std::move(id)); } future<> xenfront_net_device::queue_rx_packet() { - printf("Got at queue\n"); auto rsp_cons = _rx_ring.rsp_cons; rmb(); auto rsp_prod = _rx_ring._sring->rsp_prod; @@ -189,27 +194,35 @@ future<> xenfront_net_device::queue_rx_packet() { } _rx_ring._sring->rsp_event = rsp_cons + 1; - _rx_ring._sring->req_prod = rsp_cons + 1; rsp_prod = _rx_ring._sring->rsp_prod; - // FIXME: END GRANT. FIXME: ALLOCATE MORE MEMORY + + _rx_refs->free_ref(entry); + _rx_ring.entries.free_index(rsp.id).then([this, id = rsp.id]() { + alloc_one_rx_reference(id); + }); } // FIXME: Queue_rx maybe should not be a future then return make_ready_future<>(); } +void xenfront_net_device::alloc_one_rx_reference(unsigned index) { + + _rx_ring.entries[index] = _rx_refs->new_ref(); + + // This is how the backend knows where to put data. + auto req = &_rx_ring._sring->_ring[index].req; + req->id = index; + req->gref = _rx_ring.entries[index].xen_id; +} + future<> xenfront_net_device::alloc_rx_references(unsigned refs) { auto req_prod = _rx_ring.req_prod_pvt; rmb(); for (auto i = req_prod; (i < _rx_ring.nr_ents) && (i < refs); ++i) { - _rx_ring.entries[i] = _rx_refs->new_ref(); - - // This is how the backend knows where to put data. - auto req = &_rx_ring._sring->_ring[i].req; - req->id = i; - req->gref = _rx_ring.entries[i].xen_id; + alloc_one_rx_reference(i); ++req_prod; } @@ -221,6 +234,30 @@ future<> xenfront_net_device::alloc_rx_references(unsigned refs) { return make_ready_future(); } +future<> xenfront_net_device::handle_tx_completions() { + auto prod = _tx_ring._sring->rsp_prod; + rmb(); + + for (unsigned i = _tx_ring.rsp_cons; i != prod; i++) { + auto rsp = _tx_ring[i].rsp; + + if (rsp.status == 1) { + continue; + } + + if (rsp.status != 0) { + printf("Packet error: Handle it\n"); + continue; + } + + auto entry = _tx_ring.entries[rsp.id]; + _tx_refs->free_ref(entry); + _tx_ring.entries.free_index(rsp.id).then([this]() {}); + } + _tx_ring.rsp_cons = prod; + return make_ready_future<>(); +} + ethernet_address xenfront_net_device::hw_address() { return _hw_address; } @@ -254,8 +291,8 @@ xenfront_net_device::xenfront_net_device(boost::program_options::variables_map o , _rx_evtchn(bind_rx_evtchn()) , _tx_ring(_gntalloc->alloc_ref()) , _rx_ring(_gntalloc->alloc_ref()) - , _tx_refs(_gntalloc->alloc_ref(front_ring::nr_ents, false)) - , _rx_refs(_gntalloc->alloc_ref(front_ring::nr_ents, true)) + , _tx_refs(_gntalloc->alloc_ref(front_ring::nr_ents)) + , _rx_refs(_gntalloc->alloc_ref(front_ring::nr_ents)) , _hw_address(net::parse_ethernet_address(_xenstore->read(path("mac")))) { _rx_stream.started(); @@ -292,6 +329,11 @@ xenfront_net_device::xenfront_net_device(boost::program_options::variables_map o } alloc_rx_references(_rx_ring.nr_ents); + keep_doing([this] () { + return _evtchn->pending(_tx_evtchn).then([this] { + handle_tx_completions(); + }); + }); } xenfront_net_device::~xenfront_net_device() { diff --git a/net/xenfront.hh b/net/xenfront.hh index 8a8114697d..04ebb1681c 100644 --- a/net/xenfront.hh +++ b/net/xenfront.hh @@ -9,6 +9,7 @@ #include "net.hh" #include "core/sstring.hh" #include "core/xen/gntalloc.hh" +#include "core/queue.hh" std::unique_ptr create_xenfront_net_device(boost::program_options::variables_map opts, bool userspace); boost::program_options::options_description get_xenfront_net_options_description(); @@ -71,13 +72,21 @@ template class front_ring { public: class entries { + protected: + queue _ids; private: - std::array _entries; + std::array::nr_ents> _entries; front_ring *_ring; public: - entries(front_ring *ring) : _ring(ring) {} + entries(front_ring *ring) : _ids(front_ring::nr_ents), _ring(ring) { + for (unsigned i = 0; i < front_ring::nr_ents; ++i) { + _ids.push(std::move(i)); + } + } gntref& operator[](std::size_t i) { return _entries[_ring->idx(i)]; } friend front_ring; + future get_index(); + future<> free_index(unsigned index); }; protected: uint32_t idx(int i) { return i & (nr_ents - 1); } @@ -92,8 +101,6 @@ public: , _sring(new (r.page) sring()) { } - - future free_idx(); entries entries; sring *_sring;