mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-05-01 21:55:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

net: use a redirection table to forward packets to proxy queues

Browse Source 
Build a 128-entry redirection table to select which cpu services which
packet, when we have more cores than queues (and thus need to dispatch
internally).

Add a --hw-queue-weight to control the relative weight of the hardware queue.
With a weight of 0, the core that services the hardware queue will not
process any packets; with a weight of 1 (default) it will process an equal
share of packets, compared to proxy queues.
This commit is contained in:
Avi Kivity
2015-01-20 18:16:58 +02:00
parent 285d4af077
commit 5678a0995e
2 changed files with 44 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
net/native-stack.cc
View File

@@ -80,9 +80,12 @@ void create_native_net_device(boost::program_options::variables_map opts) {
uint16_t qid = engine.cpu_id();
if (qid < sdev->hw_queues_count()) {
auto qp = sdev->init_local_queue(opts, qid);
std::map<unsigned, float> cpu_weights;
for (unsigned i = sdev->hw_queues_count() + qid % sdev->hw_queues_count(); i < smp::count; i+= sdev->hw_queues_count()) {
qp->add_proxy(i);
cpu_weights[i] = 1;
}
cpu_weights[qid] = opts["hw-queue-weight"].as<float>();
qp->configure_proxies(cpu_weights);
sdev->set_local_queue(std::move(qp));
} else {
auto master = qid % sdev->hw_queues_count();
@@ -299,6 +302,9 @@ boost::program_options::options_description nns_options() {
("dhcp",
boost::program_options::value<bool>()->default_value(true),
"Use DHCP discovery")
("hw-queue-weight",
boost::program_options::value<float>()->default_value(1.0f),
"Weighing of a hardware network queue relative to a software queue (0=no work, 1=equal share)")
#ifdef HAVE_DPDK
("dpdk-pmd", "Use DPDK PMD drivers")
#endif

53
net/net.hh
View File

@@ -117,7 +117,7 @@ public:
class qp {
using packet_provider_type = std::function<std::experimental::optional<packet> ()>;
std::vector<packet_provider_type> _pkt_providers;
std::vector<unsigned> proxies;
std::experimental::optional<std::array<uint8_t, 128>> _sw_reta;
circular_buffer<packet> _proxy_packetq;
stream<packet> _rx_stream;
reactor::poller _tx_poller;
@@ -173,19 +173,40 @@ public:
return sent;
}
virtual void rx_start() {};
bool may_forward() { return !proxies.empty(); }
void add_proxy(unsigned cpu) {
if(proxies.empty()) {
register_packet_provider([this] {
std::experimental::optional<packet> p;
if (!_proxy_packetq.empty()) {
p = std::move(_proxy_packetq.front());
_proxy_packetq.pop_front();
}
return p;
});
void configure_proxies(const std::map<unsigned, float>& cpu_weights) {
assert(!cpu_weights.empty());
if ((cpu_weights.size() == 1 && cpu_weights.begin()->first == engine.cpu_id())) {
// special case queue sending to self only, to avoid requiring a hash value
return;
}
proxies.push_back(cpu);
register_packet_provider([this] {
std::experimental::optional<packet> p;
if (!_proxy_packetq.empty()) {
p = std::move(_proxy_packetq.front());
_proxy_packetq.pop_front();
}
return p;
});
build_sw_reta(cpu_weights);
}
// build REdirection TAble for cpu_weights map: target cpu -> weight
void build_sw_reta(const std::map<unsigned, float>& cpu_weights) {
float total_weight = 0;
for (auto&& x : cpu_weights) {
total_weight += x.second;
}
float accum = 0;
unsigned idx = 0;
std::array<uint8_t, 128> reta;
for (auto&& entry : cpu_weights) {
auto cpu = entry.first;
auto weight = entry.second;
accum += weight;
while (idx < (accum / total_weight * reta.size() - 0.5)) {
reta[idx++] = cpu;
}
}
_sw_reta = reta;
}
void proxy_send(packet p) {
_proxy_packetq.push_back(std::move(p));
@@ -256,12 +277,12 @@ public:
template <typename Func>
unsigned forward_dst(unsigned src_cpuid, Func&& hashfn) {
auto& qp = queue_for_cpu(src_cpuid);
if (!qp.may_forward()) {
if (!qp._sw_reta) {
return src_cpuid;
}
auto hash = hashfn() >> _rss_table_bits;
auto idx = hash % (qp.proxies.size() + 1);
return idx ? qp.proxies[idx - 1] : src_cpuid;
auto& reta = *qp._sw_reta;
return reta[hash % reta.size()];
}
virtual unsigned hash2cpu(uint32_t hash) {
// there is an assumption here that qid == cpu_id which will