From 5678a0995ec6112e91c5cc8b3d9dc2d590e5ce44 Mon Sep 17 00:00:00 2001
From: Avi Kivity <avi@cloudius-systems.com>
Date: Tue, 20 Jan 2015 18:16:58 +0200
Subject: [PATCH] net: use a redirection table to forward packets to proxy
 queues

Build a 128-entry redirection table to select which cpu services which
packet, when we have more cores than queues (and thus need to dispatch
internally).

Add a --hw-queue-weight to control the relative weight of the hardware queue.
With a weight of 0, the core that services the hardware queue will not
process any packets; with a weight of 1 (default) it will process an equal
share of packets, compared to proxy queues.
---
 net/native-stack.cc |  8 ++++++-
 net/net.hh          | 53 +++++++++++++++++++++++++++++++--------------
 2 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/net/native-stack.cc b/net/native-stack.cc
index 53794f1986..1bd542678b 100644
--- a/net/native-stack.cc
+++ b/net/native-stack.cc
@@ -80,9 +80,12 @@ void create_native_net_device(boost::program_options::variables_map opts) {
             uint16_t qid = engine.cpu_id();
             if (qid < sdev->hw_queues_count()) {
                 auto qp = sdev->init_local_queue(opts, qid);
+                std::map<unsigned, float> cpu_weights;
                 for (unsigned i = sdev->hw_queues_count() + qid % sdev->hw_queues_count(); i < smp::count; i+= sdev->hw_queues_count()) {
-                    qp->add_proxy(i);
+                    cpu_weights[i] = 1;
                 }
+                cpu_weights[qid] = opts["hw-queue-weight"].as<float>();
+                qp->configure_proxies(cpu_weights);
                 sdev->set_local_queue(std::move(qp));
             } else {
                 auto master = qid % sdev->hw_queues_count();
@@ -299,6 +302,9 @@ boost::program_options::options_description nns_options() {
         ("dhcp",
                 boost::program_options::value<bool>()->default_value(true),
                         "Use DHCP discovery")
+        ("hw-queue-weight",
+                boost::program_options::value<float>()->default_value(1.0f),
+                "Weighing of a hardware network queue relative to a software queue (0=no work, 1=equal share)")
 #ifdef HAVE_DPDK
         ("dpdk-pmd", "Use DPDK PMD drivers")
 #endif
diff --git a/net/net.hh b/net/net.hh
index 0254a9e8dc..e9b57d6c78 100644
--- a/net/net.hh
+++ b/net/net.hh
@@ -117,7 +117,7 @@ public:
 class qp {
     using packet_provider_type = std::function<std::experimental::optional<packet> ()>;
     std::vector<packet_provider_type> _pkt_providers;
-    std::vector<unsigned> proxies;
+    std::experimental::optional<std::array<uint8_t, 128>> _sw_reta;
     circular_buffer<packet> _proxy_packetq;
     stream<packet> _rx_stream;
     reactor::poller _tx_poller;
@@ -173,19 +173,40 @@ public:
         return sent;
     }
     virtual void rx_start() {};
-    bool may_forward() { return !proxies.empty(); }
-    void add_proxy(unsigned cpu) {
-        if(proxies.empty()) {
-            register_packet_provider([this] {
-                std::experimental::optional<packet> p;
-                if (!_proxy_packetq.empty()) {
-                    p = std::move(_proxy_packetq.front());
-                    _proxy_packetq.pop_front();
-                }
-                return p;
-            });
+    void configure_proxies(const std::map<unsigned, float>& cpu_weights) {
+        assert(!cpu_weights.empty());
+        if ((cpu_weights.size() == 1 && cpu_weights.begin()->first == engine.cpu_id())) {
+            // special case queue sending to self only, to avoid requiring a hash value
+            return;
         }
-        proxies.push_back(cpu);
+        register_packet_provider([this] {
+            std::experimental::optional<packet> p;
+            if (!_proxy_packetq.empty()) {
+                p = std::move(_proxy_packetq.front());
+                _proxy_packetq.pop_front();
+            }
+            return p;
+        });
+        build_sw_reta(cpu_weights);
+    }
+    // build REdirection TAble for cpu_weights map: target cpu -> weight
+    void build_sw_reta(const std::map<unsigned, float>& cpu_weights) {
+        float total_weight = 0;
+        for (auto&& x : cpu_weights) {
+            total_weight += x.second;
+        }
+        float accum = 0;
+        unsigned idx = 0;
+        std::array<uint8_t, 128> reta;
+        for (auto&& entry : cpu_weights) {
+            auto cpu = entry.first;
+            auto weight = entry.second;
+            accum += weight;
+            while (idx < (accum / total_weight * reta.size() - 0.5)) {
+                reta[idx++] = cpu;
+            }
+        }
+        _sw_reta = reta;
     }
     void proxy_send(packet p) {
         _proxy_packetq.push_back(std::move(p));
@@ -256,12 +277,12 @@ public:
     template <typename Func>
     unsigned forward_dst(unsigned src_cpuid, Func&& hashfn) {
         auto& qp = queue_for_cpu(src_cpuid);
-        if (!qp.may_forward()) {
+        if (!qp._sw_reta) {
             return src_cpuid;
         }
         auto hash = hashfn() >> _rss_table_bits;
-        auto idx = hash % (qp.proxies.size() + 1);
-        return idx ? qp.proxies[idx - 1] : src_cpuid;
+        auto& reta = *qp._sw_reta;
+        return reta[hash % reta.size()];
     }
     virtual unsigned hash2cpu(uint32_t hash) {
         // there is an assumption here that qid == cpu_id which will