From c40750f6e4b0356881675201bbba2d47511c83de Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Thu, 10 Aug 2017 14:59:34 -0400 Subject: [PATCH 001/156] dht: use proper value type to compute expiration --- src/dht.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index 901655be..49e21a66 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -1267,9 +1267,8 @@ Dht::storageStore(const InfoHash& id, const Sp& value, time_point created { const auto& now = scheduler.time(); created = std::min(created, now); - auto expiration = created + getType(value->id).expiration; - - if ( expiration < now ) + auto expiration = created + getType(value->type).expiration; + if (expiration < now) return false; auto st = store.find(id); From a1d67b881b3b8107917d6632222284b5c92edcd9 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Fri, 11 Aug 2017 12:43:42 -0400 Subject: [PATCH 002/156] python: fix invalid pointer access --- python/opendht.pyx | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/python/opendht.pyx b/python/opendht.pyx index 870e10ed..87cb1c19 100644 --- a/python/opendht.pyx +++ b/python/opendht.pyx @@ -179,8 +179,11 @@ cdef class Query(object): cdef class Select(object): cdef cpp.Select _select - def __cinit__(self, str q_str=''): - self._select = cpp.Select(q_str.encode()) + def __cinit__(self, str q_str=None): + if q_str: + self._select = cpp.Select(q_str.encode()) + else: + self._select = cpp.Select() def __str__(self): return self._select.toString().decode() def isSatisfiedBy(self, Select os): @@ -191,8 +194,11 @@ cdef class Select(object): cdef class Where(object): cdef cpp.Where _where - def __cinit__(self, str q_str=''): - self._where = cpp.Where(q_str.encode()) + def __cinit__(self, str q_str=None): + if q_str: + self._where = cpp.Where(q_str.encode()) + else: + self._where = cpp.Where() def __str__(self): return self._where.toString().decode() def isSatisfiedBy(self, Where where): @@ -527,6 +533,8 @@ cdef class DhtRunner(_WithID): if get_cb: cb_obj = {'get':get_cb, 'done':done_cb, 'filter':filter} ref.Py_INCREF(cb_obj) + if where is None: + where = Where() self.thisptr.get().get(key._infohash, cpp.bindGetCb(get_callback, cb_obj), cpp.bindDoneCb(done_callback, cb_obj), cpp.nullptr, #filter implemented in the get_callback @@ -654,4 +662,3 @@ cdef class Pht(object): val, cpp.bindDoneCbSimple(done_callback_simple, cb_obj) ) - From 3bdaf2e20bf2d9ffd2e83e5a50c334c4a4eed0a2 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Fri, 11 Aug 2017 14:54:31 -0400 Subject: [PATCH 003/156] update to version 1.3.6 Bugfix release. --- CMakeLists.txt | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f607dbb..3eeaf38c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required (VERSION 3.1) project (opendht) set (opendht_VERSION_MAJOR 1) -set (opendht_VERSION_MINOR 3.5) +set (opendht_VERSION_MINOR 3.6) set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR}) set (PACKAGE_VERSION ${opendht_VERSION}) set (VERSION "${opendht_VERSION}") diff --git a/configure.ac b/configure.ac index dbc5359c..3a4b8eaf 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl define macros m4_define([opendht_major_version], 1) m4_define([opendht_minor_version], 3) -m4_define([opendht_patch_version], 5) +m4_define([opendht_patch_version], 6) m4_define([opendht_version], [opendht_major_version.opendht_minor_version.opendht_patch_version]) From 1b9e9452c4382865ef61608d7b5d5f8f37185f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Sun, 27 Aug 2017 16:40:39 +0200 Subject: [PATCH 004/156] NodeExport: add msgpack serialization --- include/opendht/infohash.h | 15 ++++++++++++++- src/infohash.cpp | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index 74186ed2..ed50bc9c 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -299,10 +299,23 @@ public: static constexpr const InfoHash zeroes {}; -struct NodeExport { +struct OPENDHT_PUBLIC NodeExport { InfoHash id; sockaddr_storage ss; socklen_t sslen; + + template + void msgpack_pack(Packer& pk) const + { + pk.pack_map(2); + pk.pack(std::string("id")); + pk.pack(id); + pk.pack(std::string("addr")); + pk.pack_bin(sslen); + pk.pack_bin_body((char*)&ss, sslen); + } + + void msgpack_unpack(msgpack::object o); }; } diff --git a/src/infohash.cpp b/src/infohash.cpp index 7631a9d5..7f269efd 100644 --- a/src/infohash.cpp +++ b/src/infohash.cpp @@ -121,4 +121,25 @@ std::istream& operator>> (std::istream& s, InfoHash& h) return s; } +void +NodeExport::msgpack_unpack(msgpack::object o) +{ + if (o.type != msgpack::type::MAP) + throw msgpack::type_error(); + if (o.via.map.size < 2) + throw msgpack::type_error(); + if (o.via.map.ptr[0].key.as() != "id") + throw msgpack::type_error(); + if (o.via.map.ptr[1].key.as() != "addr") + throw msgpack::type_error(); + const auto& addr = o.via.map.ptr[1].val; + if (addr.type != msgpack::type::BIN) + throw msgpack::type_error(); + if (addr.via.bin.size > sizeof(sockaddr_storage)) + throw msgpack::type_error(); + id.msgpack_unpack(o.via.map.ptr[0].val); + sslen = addr.via.bin.size; + std::copy_n(addr.via.bin.ptr, addr.via.bin.size, (char*)&ss); +} + } From bedb51bf52cbd6e07737211f2d38045342f9216e Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Thu, 10 Aug 2017 09:47:41 -0400 Subject: [PATCH 005/156] crypto: add EC curve identity generation --- include/opendht/crypto.h | 3 +++ src/crypto.cpp | 17 ++++++++++++++++- tools/dhtnode.cpp | 4 ++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/include/opendht/crypto.h b/include/opendht/crypto.h index 81985c66..07ca9199 100644 --- a/include/opendht/crypto.h +++ b/include/opendht/crypto.h @@ -583,6 +583,9 @@ using SecureBlob = secure_vector; OPENDHT_PUBLIC Identity generateIdentity(const std::string& name, Identity ca, unsigned key_length, bool is_ca); OPENDHT_PUBLIC Identity generateIdentity(const std::string& name = "dhtnode", Identity ca = {}, unsigned key_length = 4096); +OPENDHT_PUBLIC Identity generateEcIdentity(const std::string& name, Identity ca, bool is_ca); +OPENDHT_PUBLIC Identity generateEcIdentity(const std::string& name = "dhtnode", Identity ca = {}); + /** * Performs SHA512, SHA256 or SHA1, depending on hash_length. diff --git a/src/crypto.cpp b/src/crypto.cpp index 8b162138..dae67deb 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -882,6 +882,19 @@ generateIdentity(const std::string& name, Identity ca, unsigned key_length) { return generateIdentity(name, ca, key_length, !ca.first || !ca.second); } +Identity +generateEcIdentity(const std::string& name, crypto::Identity ca, bool is_ca) +{ + auto key = std::make_shared(PrivateKey::generateEC()); + auto cert = std::make_shared(Certificate::generate(*key, name, ca, is_ca)); + return {std::move(key), std::move(cert)}; +} + +Identity +generateEcIdentity(const std::string& name, Identity ca) { + return generateEcIdentity(name, ca, !ca.first || !ca.second); +} + Certificate Certificate::generate(const PrivateKey& key, const std::string& name, Identity ca, bool is_ca) { @@ -916,10 +929,12 @@ Certificate::generate(const PrivateKey& key, const std::string& name, Identity c gnutls_x509_crt_set_serial(cert, &cert_serial, sizeof(cert_serial)); } - unsigned key_usage = GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_DATA_ENCIPHERMENT; + unsigned key_usage = 0; if (is_ca) { gnutls_x509_crt_set_ca_status(cert, 1); key_usage |= GNUTLS_KEY_KEY_CERT_SIGN | GNUTLS_KEY_CRL_SIGN; + } else { + key_usage |= GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_DATA_ENCIPHERMENT; } gnutls_x509_crt_set_key_usage(cert, key_usage); diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 8aaf75f3..7e42c641 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -365,8 +365,8 @@ main(int argc, char **argv) dht::crypto::Identity crt {}; if (params.generate_identity) { - auto ca_tmp = dht::crypto::generateIdentity("DHT Node CA"); - crt = dht::crypto::generateIdentity("DHT Node", ca_tmp); + auto ca_tmp = dht::crypto::generateEcIdentity("DHT Node CA"); + crt = dht::crypto::generateEcIdentity("DHT Node", ca_tmp); } dht->run(params.port, crt, true, params.network); From 1d84b34cf6b5944c5e1b48761c76c780fd05bc56 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Wed, 30 Aug 2017 10:46:30 +0200 Subject: [PATCH 006/156] value: avoid checking failbit directly failbit is implementation defined, use the bool operator instead. --- src/value.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/value.cpp b/src/value.cpp index 18456390..101165fd 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -360,10 +360,10 @@ Where::Where(const std::string& q_str) { std::string s {}; std::istringstream convert {value_str}; convert >> v; - if (convert.failbit + if (not convert and value_str.size() > 1 - and value_str[0] == '\"' - and value_str[value_str.size()-1] == '\"') + and value_str[0] == '"' + and value_str[value_str.size()-1] == '"') s = value_str.substr(1, value_str.size()-2); else s = value_str; @@ -490,4 +490,3 @@ std::ostream& operator<<(std::ostream& s, const dht::Where& where) { } - From a82f5752320a03711465c23ee284b9669dcf6521 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Wed, 30 Aug 2017 10:48:49 +0200 Subject: [PATCH 007/156] NodeExport: add ostream operator --- include/opendht/infohash.h | 2 ++ src/infohash.cpp | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index ed50bc9c..57920b5b 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -316,6 +316,8 @@ struct OPENDHT_PUBLIC NodeExport { } void msgpack_unpack(msgpack::object o); + + OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const InfoHash& h); }; } diff --git a/src/infohash.cpp b/src/infohash.cpp index 7f269efd..3c4e0eaf 100644 --- a/src/infohash.cpp +++ b/src/infohash.cpp @@ -142,4 +142,10 @@ NodeExport::msgpack_unpack(msgpack::object o) std::copy_n(addr.via.bin.ptr, addr.via.bin.size, (char*)&ss); } +std::ostream& operator<< (std::ostream& s, const NodeExport& h) +{ + msgpack::pack(s, h); + return s; +} + } From 1c45f342625a19746ba089999bef8dadd3cadf25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 26 Sep 2017 16:30:29 +0200 Subject: [PATCH 008/156] SockAddr: use pointer to sockaddr Save memory and reduce copies by using a pointer to a sockaddr structure of the right size instead of using a full sockaddr_storage structure for every address. --- include/opendht/dhtrunner.h | 2 +- include/opendht/network_engine.h | 30 +------- include/opendht/node.h | 10 ++- include/opendht/sockaddr.h | 106 ++++++++++++++++++-------- src/dht.cpp | 10 +-- src/dhtrunner.cpp | 26 ++++--- src/network_engine.cpp | 127 +++++++++++++++---------------- src/utils.cpp | 7 +- 8 files changed, 167 insertions(+), 151 deletions(-) diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index 0007c31a..fd3346c0 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -250,7 +250,7 @@ public: * @param f: address family of the bound port to retreive. */ in_port_t getBoundPort(sa_family_t f = AF_INET) const { - return ntohs(((sockaddr_in*)&getBound(f).first)->sin_port); + return getBound(f).getPort(); } std::pair getStoreSize() const; diff --git a/include/opendht/network_engine.h b/include/opendht/network_engine.h index 8cf6827a..dee6584b 100644 --- a/include/opendht/network_engine.h +++ b/include/opendht/network_engine.h @@ -137,7 +137,7 @@ private: /** * Called when an addres is reported from a requested node. * - * @param h: id + * @param h: id * @param saddr_len (type: socklen_t) lenght of the sockaddr struct. */ std::function onReportedAddr; @@ -553,34 +553,6 @@ private: NodeCache cache {}; - /** - * A comparator to classify IP addresses, only considering the - * first 64 bits in IPv6. - */ - struct cmpSockAddr { - bool operator()(const SockAddr& a, const SockAddr& b) const { - if (a.second != b.second) - return a.second < b.second; - socklen_t start, len; - switch(a.getFamily()) { - case AF_INET: - start = offsetof(sockaddr_in, sin_addr); - len = sizeof(in_addr); - break; - case AF_INET6: - start = offsetof(sockaddr_in6, sin6_addr); - // don't consider more than 64 bits (IPv6) - len = 8; - break; - default: - start = 0; - len = a.second; - break; - } - - return std::memcmp((uint8_t*)&a.first+start, (uint8_t*)&b.first+start, len) < 0; - } - }; // global limiting should be triggered by at least 8 different IPs using IpLimiter = RateLimiter; using IpLimiterMap = std::map; diff --git a/include/opendht/node.h b/include/opendht/node.h index 7a2c9256..15791a41 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -46,7 +46,7 @@ struct Node { return id; } std::pair getAddr() const { - return {(const sockaddr*)&addr.first, addr.second}; + return {addr.get(), addr.getLength()}; } std::string getAddrStr() const { return addr.toString(); @@ -68,7 +68,13 @@ struct Node { bool isPendingMessage() const; size_t getPendingMessageCount() const; - NodeExport exportNode() const { return NodeExport {id, addr.first, addr.second}; } + NodeExport exportNode() const { + NodeExport ne; + ne.id = id; + ne.sslen = addr.getLength(); + std::memcpy(&ne.ss, addr.get(), ne.sslen); + return ne; + } sa_family_t getFamily() const { return addr.getFamily(); } void update(const SockAddr&); diff --git a/include/opendht/sockaddr.h b/include/opendht/sockaddr.h index d2b814d1..070a6698 100644 --- a/include/opendht/sockaddr.h +++ b/include/opendht/sockaddr.h @@ -34,54 +34,91 @@ typedef uint16_t sa_family_t; typedef uint16_t in_port_t; #endif +#include +#include + +#include +#include + namespace dht { OPENDHT_PUBLIC std::string print_addr(const sockaddr* sa, socklen_t slen); OPENDHT_PUBLIC std::string print_addr(const sockaddr_storage& ss, socklen_t sslen); -struct OPENDHT_PUBLIC SockAddr : public std::pair { +class OPENDHT_PUBLIC SockAddr { public: - SockAddr() : pair::pair({},0) {} - SockAddr(const SockAddr& o) : pair::pair({},o.second) { - std::copy_n((uint8_t*)&o.first, o.second, (uint8_t*)&first); + SockAddr() {} + SockAddr(const SockAddr& o) { + len = o.len; + if (len) { + addr.reset((sockaddr*)std::malloc(len)); + std::memcpy((uint8_t*)addr.get(), (const uint8_t*)o.get(), len); + } else + addr.reset(); } - SockAddr(const sockaddr* sa, socklen_t len) : pair::pair({},len) { - if (len > sizeof(sockaddr_storage)) + SockAddr(SockAddr&& o) : len(o.len), addr(std::move(o.addr)) { + o.len = 0; + } + SockAddr(const sockaddr* sa, socklen_t length) { + if (length > sizeof(sockaddr_storage)) throw std::runtime_error("Socket address length is too large"); - std::copy_n((uint8_t*)sa, len, (uint8_t*)&first); + len = length; + addr.reset((sockaddr*)std::malloc(len)); + std::memcpy((uint8_t*)get(), (const uint8_t*)sa, len); } SockAddr(const sockaddr_storage& ss, socklen_t len) : SockAddr((const sockaddr*)&ss, len) {} bool operator<(const SockAddr& o) const { - if (second != o.second) - return second < o.second; - return std::memcmp((uint8_t*)&first, (uint8_t*)&o.first, second) < 0; + if (len != o.len) + return len < o.len; + return std::memcmp((uint8_t*)get(), (uint8_t*)o.get(), len) < 0; } bool equals(const SockAddr& o) const { - return second == o.second - && std::memcmp((uint8_t*)&first, (uint8_t*)&o.first, second) == 0; + return len == o.len + && std::memcmp((uint8_t*)get(), (uint8_t*)o.get(), len) == 0; } SockAddr& operator=(const SockAddr& o) { - std::copy_n((const uint8_t*)&o.first, o.second, (uint8_t*)&first); - second = o.second; + if (len != o.len) { + len = o.len; + addr.reset((sockaddr*)std::realloc(addr.release(), len)); + } + std::memcpy((uint8_t*)get(), (const uint8_t*)o.get(), len); + return *this; + } + SockAddr& operator=(SockAddr&& o) { + len = o.len; + o.len = 0; + addr = std::move(o.addr); return *this; } std::string toString() const { - return print_addr(first, second); + return print_addr(get(), getLength()); } - sa_family_t getFamily() const { return second > sizeof(sa_family_t) ? first.ss_family : AF_UNSPEC; } + sa_family_t getFamily() const { return len > sizeof(sa_family_t) ? addr->sa_family : AF_UNSPEC; } void setFamily(sa_family_t af) { - first.ss_family = af; + socklen_t new_length; switch(af) { case AF_INET: - second = sizeof(sockaddr_in); + new_length = sizeof(sockaddr_in); break; case AF_INET6: - second = sizeof(sockaddr_in6); + new_length = sizeof(sockaddr_in6); break; + default: + new_length = 0; } + if (new_length != len) { + len = new_length; + if (len) { + addr.reset((sockaddr*)std::realloc(addr.release(), len)); + std::memset(addr.get(), '\0', len); + } else + addr.reset(); + } + if (len > sizeof(sa_family_t)) + addr->sa_family = af; } in_port_t getPort() const { @@ -105,17 +142,23 @@ public: } } + socklen_t getLength() const { return len; } + explicit operator bool() const noexcept { + return len; + } + + const sockaddr* get() const { return addr.get(); } const sockaddr_in& getIPv4() const { - return *reinterpret_cast(&first); + return *reinterpret_cast(get()); } const sockaddr_in6& getIPv6() const { - return *reinterpret_cast(&first); + return *reinterpret_cast(get()); } sockaddr_in& getIPv4() { - return *reinterpret_cast(&first); + return *reinterpret_cast(addr.get()); } sockaddr_in6& getIPv6() { - return *reinterpret_cast(&first); + return *reinterpret_cast(addr.get()); } /** @@ -136,8 +179,8 @@ public: */ struct ipCmp { bool operator()(const SockAddr& a, const SockAddr& b) const { - if (a.second != b.second) - return a.second < b.second; + if (a.len != b.len) + return a.len < b.len; socklen_t start, len; switch(a.getFamily()) { case AF_INET: @@ -151,18 +194,19 @@ public: break; default: start = 0; - len = a.second; + len = a.len; break; } - - return std::memcmp((uint8_t*)&a.first+start, (uint8_t*)&b.first+start, len) < 0; + return std::memcmp((uint8_t*)a.get()+start, + (uint8_t*)b.get()+start, len) < 0; } }; - +private: + socklen_t len {0}; + struct free_delete { void operator()(void* p) { free(p); } }; + std::unique_ptr addr {}; }; OPENDHT_PUBLIC bool operator==(const SockAddr& a, const SockAddr& b); -OPENDHT_PUBLIC std::string printAddr(const SockAddr& addr); - } diff --git a/src/dht.cpp b/src/dht.cpp index 49e21a66..f739280b 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -120,7 +120,7 @@ Dht::getPublicAddress(sa_family_t family) }); std::vector ret; for (const auto& addr : reported_addr) - if (!family || family == addr.second.first.ss_family) + if (!family || family == addr.second.getFamily()) ret.emplace_back(addr.second); return ret; } @@ -1456,7 +1456,7 @@ Dht::makeToken(const SockAddr& addr, bool old) const bool Dht::tokenMatch(const Blob& token, const SockAddr& addr) const { - if (!addr.second || token.size() != TOKEN_SIZE) + if (not addr or token.size() != TOKEN_SIZE) return false; if (token == makeToken(addr, false)) return true; @@ -2181,7 +2181,7 @@ Dht::onReportedAddr(const InfoHash& id, const SockAddr& addr) { const auto& b = buckets(addr.getFamily()).findBucket(id); b->time = scheduler.time(); - if (addr.second) + if (addr) reportedAddr(addr); } @@ -2379,7 +2379,7 @@ Dht::onAnnounce(Sp node, DHT_LOG.w(hash, node->id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str()); } else { const auto& type = getType(lv->type); - if (type.editPolicy(hash, lv, vc, node->id, (sockaddr*)&node->addr.first, node->addr.second)) { + if (type.editPolicy(hash, lv, vc, node->id, node->addr.get(), node->addr.getLength())) { DHT_LOG.d(hash, node->id, "[store %s] editing %s", hash.toString().c_str(), vc->toString().c_str()); storageStore(hash, vc, created, &node->addr); @@ -2391,7 +2391,7 @@ Dht::onAnnounce(Sp node, } else { // Allow the value to be edited by the storage policy const auto& type = getType(vc->type); - if (type.storePolicy(hash, vc, node->id, (sockaddr*)&node->addr.first, node->addr.second)) { + if (type.storePolicy(hash, vc, node->id, node->addr.get(), node->addr.getLength())) { DHT_LOG.d(hash, node->id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str()); storageStore(hash, vc, created, &node->addr); } else { diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index d8903274..66b4ebbd 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -303,7 +303,7 @@ DhtRunner::getPublicAddressStr(sa_family_t af) { auto addrs = getPublicAddress(af); std::vector ret(addrs.size()); - std::transform(addrs.begin(), addrs.end(), ret.begin(), dht::printAddr); + std::transform(addrs.begin(), addrs.end(), ret.begin(), [](const SockAddr& a) { return a.toString(); }); return ret; } @@ -390,8 +390,10 @@ DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::C int rc = bind(s4, (sockaddr*)sin4, sizeof(sockaddr_in)); if(rc < 0) throw DhtException("Can't bind IPv4 socket on " + dht::print_addr((sockaddr*)sin4, sizeof(sockaddr_in))); - bound4.second = sizeof(bound4.first); - getsockname(s4, (sockaddr*)&bound4.first, &bound4.second); + sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + getsockname(s4, (sockaddr*)&ss, &ss_len); + bound4 = {ss, ss_len}; } } @@ -408,8 +410,10 @@ DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::C rc = bind(s6, (sockaddr*)sin6, sizeof(sockaddr_in6)); if(rc < 0) throw DhtException("Can't bind IPv6 socket on " + dht::print_addr((sockaddr*)sin6, sizeof(sockaddr_in6))); - bound6.second = sizeof(bound6.first); - getsockname(s6, (sockaddr*)&bound6.first, &bound6.second); + sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + getsockname(s6, (sockaddr*)&ss, &ss_len); + bound6 = {ss, ss_len}; } } #endif @@ -441,19 +445,19 @@ DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::C if(rc > 0) { std::array buf; - SockAddr from; - from.second = sizeof(from.first); + sockaddr_storage from; + socklen_t from_len = sizeof(from); if(s4 >= 0 && FD_ISSET(s4, &readfds)) - rc = recvfrom(s4, (char*)buf.data(), buf.size(), 0, (struct sockaddr*)&from.first, &from.second); + rc = recvfrom(s4, (char*)buf.data(), buf.size(), 0, (sockaddr*)&from, &from_len); else if(s6 >= 0 && FD_ISSET(s6, &readfds)) - rc = recvfrom(s6, (char*)buf.data(), buf.size(), 0, (struct sockaddr*)&from.first, &from.second); + rc = recvfrom(s6, (char*)buf.data(), buf.size(), 0, (sockaddr*)&from, &from_len); else break; if (rc > 0) { { std::lock_guard lck(sock_mtx); - rcv.emplace_back(Blob {buf.begin(), buf.begin()+rc+1}, from); + rcv.emplace_back(Blob {buf.begin(), buf.begin()+rc+1}, SockAddr(from, from_len)); } cv.notify_all(); } @@ -755,7 +759,7 @@ DhtRunner::bootstrap(const SockAddr& addr, DoneCallbackSimple&& cb) { std::lock_guard lck(storage_mtx); pending_ops_prio.emplace([addr,cb](SecureDht& dht) mutable { - dht.pingNode((const sockaddr*)&addr.first, addr.second, std::move(cb)); + dht.pingNode(addr.get(), addr.getLength(), std::move(cb)); }); cv.notify_all(); } diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 56b821f2..8f23c361 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -372,7 +372,7 @@ NetworkEngine::isMartian(const SockAddr& addr) ((address[0] & 0xE0) == 0xE0); } case AF_INET6: { - if (addr.second < sizeof(sockaddr_in6)) + if (addr.getLength() < sizeof(sockaddr_in6)) return true; const auto& sin6 = addr.getIPv6(); const uint8_t* address = (const uint8_t*)&sin6.sin6_addr; @@ -651,10 +651,10 @@ packToken(msgpack::packer& pk, const Blob& token) void insertAddr(msgpack::packer& pk, const SockAddr& addr) { - size_t addr_len = std::min(addr.second, + size_t addr_len = std::min(addr.getLength(), (addr.getFamily() == AF_INET) ? sizeof(in_addr) : sizeof(in6_addr)); - void* addr_ptr = (addr.getFamily() == AF_INET) ? (void*)&((sockaddr_in*)&addr.first)->sin_addr - : (void*)&((sockaddr_in6*)&addr.first)->sin6_addr; + void* addr_ptr = (addr.getFamily() == AF_INET) ? (void*)&addr.getIPv4().sin_addr + : (void*)&addr.getIPv6().sin6_addr; pk.pack("sa"); pk.pack_bin(addr_len); pk.pack_bin_body((char*)addr_ptr, addr_len); @@ -663,7 +663,7 @@ insertAddr(msgpack::packer& pk, const SockAddr& addr) int NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr) { - if (addr.second == 0) + if (not addr) return EFAULT; int s; @@ -676,7 +676,7 @@ NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr if (s < 0) return EAFNOSUPPORT; - if (sendto(s, buf, len, flags, (const sockaddr*)&addr.first, addr.second) == -1) { + if (sendto(s, buf, len, flags, addr.get(), addr.getLength()) == -1) { int err = errno; DHT_LOG.e("Can't send message to %s: %s", addr.toString().c_str(), strerror(err)); return err; @@ -839,20 +839,18 @@ NetworkEngine::sendGetValues(Sp n, const InfoHash& info_hash, const Query& SockAddr deserializeIPv4(const uint8_t* ni) { SockAddr addr; - auto& sin = addr.getIPv4(); - std::fill_n((uint8_t*)&sin, sizeof(sockaddr_in), 0); addr.setFamily(AF_INET); - memcpy(&sin.sin_addr, ni, 4); - memcpy(&sin.sin_port, ni + 4, 2); + auto& sin = addr.getIPv4(); + std::memcpy(&sin.sin_addr, ni, 4); + std::memcpy(&sin.sin_port, ni + 4, 2); return addr; } SockAddr deserializeIPv6(const uint8_t* ni) { SockAddr addr; - auto& sin6 = addr.getIPv6(); - std::fill_n((uint8_t*)&sin6, sizeof(sockaddr_in6), 0); addr.setFamily(AF_INET6); - memcpy(&sin6.sin6_addr, ni, 16); - memcpy(&sin6.sin6_port, ni + 16, 2); + auto& sin6 = addr.getIPv6(); + std::memcpy(&sin6.sin6_addr, ni, 16); + std::memcpy(&sin6.sin6_port, ni + 16, 2); return addr; } @@ -860,41 +858,40 @@ void NetworkEngine::deserializeNodes(ParsedMessage& msg, const SockAddr& from) { if (msg.nodes4_raw.size() % NODE4_INFO_BUF_LEN != 0 || msg.nodes6_raw.size() % NODE6_INFO_BUF_LEN != 0) { throw DhtProtocolException {DhtProtocolException::WRONG_NODE_INFO_BUF_LEN}; - } else { - // deserialize nodes - const auto& now = scheduler.time(); - for (unsigned i = 0; i < msg.nodes4_raw.size() / NODE4_INFO_BUF_LEN; i++) { - uint8_t *ni = msg.nodes4_raw.data() + i * NODE4_INFO_BUF_LEN; - const InfoHash& ni_id = *reinterpret_cast(ni); - if (ni_id == myid) - continue; - SockAddr addr = deserializeIPv4(ni + ni_id.size()); - if (addr.isLoopback() and from.getFamily() == AF_INET) { - auto port = addr.getPort(); - addr = from; - addr.setPort(port); - } - if (isMartian(addr) || isNodeBlacklisted(addr)) - continue; - msg.nodes4.emplace_back(cache.getNode(ni_id, addr, now, false)); - onNewNode(msg.nodes4.back(), 0); + } + // deserialize nodes + const auto& now = scheduler.time(); + for (unsigned i = 0, n = msg.nodes4_raw.size() / NODE4_INFO_BUF_LEN; i < n; i++) { + const uint8_t* ni = msg.nodes4_raw.data() + i * NODE4_INFO_BUF_LEN; + const auto& ni_id = *reinterpret_cast(ni); + if (ni_id == myid) + continue; + SockAddr addr = deserializeIPv4(ni + ni_id.size()); + if (addr.isLoopback() and from.getFamily() == AF_INET) { + auto port = addr.getPort(); + addr = from; + addr.setPort(port); } - for (unsigned i = 0; i < msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i++) { - uint8_t *ni = msg.nodes6_raw.data() + i * NODE6_INFO_BUF_LEN; - const InfoHash& ni_id = *reinterpret_cast(ni); - if (ni_id == myid) - continue; - SockAddr addr = deserializeIPv6(ni + ni_id.size()); - if (addr.isLoopback() and from.getFamily() == AF_INET6) { - auto port = addr.getPort(); - addr = from; - addr.setPort(port); - } - if (isMartian(addr) || isNodeBlacklisted(addr)) - continue; - msg.nodes6.emplace_back(cache.getNode(ni_id, addr, now, false)); - onNewNode(msg.nodes6.back(), 0); + if (isMartian(addr) || isNodeBlacklisted(addr)) + continue; + msg.nodes4.emplace_back(cache.getNode(ni_id, addr, now, false)); + onNewNode(msg.nodes4.back(), 0); + } + for (unsigned i = 0, n = msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i < n; i++) { + const uint8_t* ni = msg.nodes6_raw.data() + i * NODE6_INFO_BUF_LEN; + const auto& ni_id = *reinterpret_cast(ni); + if (ni_id == myid) + continue; + SockAddr addr = deserializeIPv6(ni + ni_id.size()); + if (addr.isLoopback() and from.getFamily() == AF_INET6) { + auto port = addr.getPort(); + addr = from; + addr.setPort(port); } + if (isMartian(addr) || isNodeBlacklisted(addr)) + continue; + msg.nodes6.emplace_back(cache.getNode(ni_id, addr, now, false)); + onNewNode(msg.nodes6.back(), 0); } } @@ -1022,21 +1019,21 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, std::vectorsin_addr, sizeof(in_addr)); - memcpy(dest + HASH_LEN + sizeof(in_addr), &sin->sin_port, sizeof(in_port_t)); + memcpy(dest + HASH_LEN, &sin.sin_addr, sizeof(in_addr)); + memcpy(dest + HASH_LEN + sizeof(in_addr), &sin.sin_port, sizeof(in_port_t)); } } else if (af == AF_INET6) { bnodes.resize(NODE6_INFO_BUF_LEN * nnode); for (size_t i=0; isin6_addr, sizeof(in6_addr)); - memcpy(dest + HASH_LEN + sizeof(in6_addr), &sin6->sin6_port, sizeof(in_port_t)); + memcpy(dest + HASH_LEN, &sin6.sin6_addr, sizeof(in6_addr)); + memcpy(dest + HASH_LEN + sizeof(in6_addr), &sin6.sin6_port, sizeof(in_port_t)); } } return bnodes; @@ -1420,22 +1417,20 @@ ParsedMessage::msgpack_unpack(msgpack::object msg) throw msgpack::type_error(); auto l = sa->via.bin.size; if (l == sizeof(in_addr)) { - auto a = (sockaddr_in*)&addr.first; - std::fill_n((uint8_t*)a, sizeof(sockaddr_in), 0); - a->sin_family = AF_INET; - a->sin_port = 0; - std::copy_n(sa->via.bin.ptr, l, (char*)&a->sin_addr); - addr.second = sizeof(sockaddr_in); + addr.setFamily(AF_INET); + auto& a = addr.getIPv4(); + std::fill_n((uint8_t*)&a, sizeof(a), 0); + a.sin_port = 0; + std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin_addr); } else if (l == sizeof(in6_addr)) { - auto a = (sockaddr_in6*)&addr.first; - std::fill_n((uint8_t*)a, sizeof(sockaddr_in6), 0); - a->sin6_family = AF_INET6; - a->sin6_port = 0; - std::copy_n(sa->via.bin.ptr, l, (char*)&a->sin6_addr); - addr.second = sizeof(sockaddr_in6); + addr.setFamily(AF_INET6); + auto& a = addr.getIPv6(); + std::fill_n((uint8_t*)&a, sizeof(a), 0); + a.sin6_port = 0; + std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin6_addr); } } else - addr.second = 0; + addr = {}; if (auto rcreated = findMapValue(req, "c")) created = from_time_t(rcreated->as()); diff --git a/src/utils.cpp b/src/utils.cpp index afc09f04..87826e1f 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -38,7 +38,7 @@ print_addr(const sockaddr* sa, socklen_t slen) out << "[" << hbuf << "]"; else out << hbuf; - if (strcmp(sbuf, "0")) + if (std::strcmp(sbuf, "0")) out << ":" << sbuf; } else out << "[invalid address]"; @@ -51,11 +51,6 @@ print_addr(const sockaddr_storage& ss, socklen_t sslen) return print_addr((const sockaddr*)&ss, sslen); } -std::string -printAddr(const SockAddr& addr) { - return print_addr((const sockaddr*)&addr.first, addr.second); -} - bool SockAddr::isUnspecified() const { From dd13ebb6d12ec905b4a975ac7ce0ca6a0fbad1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 26 Sep 2017 16:31:21 +0200 Subject: [PATCH 009/156] cmake: use private linking for private dependencies Users of the OpenDHT shared library don't need to link against GnuTLS or Nettle. --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3eeaf38c..43d6a7cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,15 +180,15 @@ if (OPENDHT_SHARED) set_target_properties (opendht PROPERTIES SOVERSION ${opendht_VERSION_MAJOR} VERSION ${opendht_VERSION}) target_compile_definitions(opendht PRIVATE OPENDHT_BUILD) if (OPENDHT_LTO) - target_link_libraries(opendht -flto -fuse-linker-plugin) + target_link_libraries(opendht PRIVATE -flto -fuse-linker-plugin) endif () if (OPENDHT_ARGON2) target_include_directories(opendht SYSTEM PRIVATE argon2) else () - target_link_libraries(opendht ${argon2_LIBRARIES}) + target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES}) target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - target_link_libraries(opendht ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) + target_link_libraries(opendht PRIVATE ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () From f17a4e66f56374e08a3b6c8f1d02a1402e3e5013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 26 Sep 2017 16:35:38 +0200 Subject: [PATCH 010/156] cmake: add flag to enable address sanitizer and stack protector --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43d6a7cc..c612daac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ option (OPENDHT_TOOLS "Build DHT tools" ON) option (OPENDHT_SYSTEMD "Install systemd module" OFF) option (OPENDHT_ARGON2 "Use included argon2 sources" OFF) option (OPENDHT_LTO "Build with LTO" OFF) +option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF) # Dependencies list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") @@ -41,6 +42,9 @@ endif () set (CMAKE_CXX_STANDARD 11) set (CMAKE_CXX_STANDARD_REQUIRED on) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden") +if (OPENDHT_SANITIZE) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fstack-protector-strong") +endif () set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMSGPACK_DISABLE_LEGACY_NIL -DMSGPACK_DISABLE_LEGACY_CONVERT") if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) From 078c4f064acc6b1688f7f995f57ab9cf50b57ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 26 Sep 2017 16:42:33 +0200 Subject: [PATCH 011/156] cmake: use Threads package --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c612daac..15b7cac2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF) # Dependencies list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") +find_package (Threads) find_package (PkgConfig) find_package (GnuTLS 3.3 REQUIRED) pkg_search_module (Nettle nettle) @@ -41,7 +42,7 @@ endif () # Build flags set (CMAKE_CXX_STANDARD 11) set (CMAKE_CXX_STANDARD_REQUIRED on) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_THREAD_LIBS_INIT} -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden") if (OPENDHT_SANITIZE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fstack-protector-strong") endif () From 131073e2118c0152829eb6dcf2dfb1db7956d636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 27 Sep 2017 11:36:42 +0200 Subject: [PATCH 012/156] SockAddr: cleanup --- include/opendht/sockaddr.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/include/opendht/sockaddr.h b/include/opendht/sockaddr.h index 070a6698..86d1d601 100644 --- a/include/opendht/sockaddr.h +++ b/include/opendht/sockaddr.h @@ -111,11 +111,8 @@ public: } if (new_length != len) { len = new_length; - if (len) { - addr.reset((sockaddr*)std::realloc(addr.release(), len)); - std::memset(addr.get(), '\0', len); - } else - addr.reset(); + if (len) addr.reset((sockaddr*)std::calloc(len, 1)); + else addr.reset(); } if (len > sizeof(sa_family_t)) addr->sa_family = af; @@ -148,6 +145,8 @@ public: } const sockaddr* get() const { return addr.get(); } + sockaddr* get() { return addr.get(); } + const sockaddr_in& getIPv4() const { return *reinterpret_cast(get()); } @@ -155,10 +154,10 @@ public: return *reinterpret_cast(get()); } sockaddr_in& getIPv4() { - return *reinterpret_cast(addr.get()); + return *reinterpret_cast(get()); } sockaddr_in6& getIPv6() { - return *reinterpret_cast(addr.get()); + return *reinterpret_cast(get()); } /** @@ -203,7 +202,7 @@ public: }; private: socklen_t len {0}; - struct free_delete { void operator()(void* p) { free(p); } }; + struct free_delete { void operator()(void* p) { std::free(p); } }; std::unique_ptr addr {}; }; From 0abd288958abf45aef83f8b3f791b8f3c763b4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 5 Oct 2017 13:54:49 +0200 Subject: [PATCH 013/156] SockAddr: factorize setters --- include/opendht/sockaddr.h | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/include/opendht/sockaddr.h b/include/opendht/sockaddr.h index 86d1d601..d23f6844 100644 --- a/include/opendht/sockaddr.h +++ b/include/opendht/sockaddr.h @@ -49,12 +49,7 @@ class OPENDHT_PUBLIC SockAddr { public: SockAddr() {} SockAddr(const SockAddr& o) { - len = o.len; - if (len) { - addr.reset((sockaddr*)std::malloc(len)); - std::memcpy((uint8_t*)addr.get(), (const uint8_t*)o.get(), len); - } else - addr.reset(); + set(o.get(), o.getLength()); } SockAddr(SockAddr&& o) : len(o.len), addr(std::move(o.addr)) { o.len = 0; @@ -62,28 +57,22 @@ public: SockAddr(const sockaddr* sa, socklen_t length) { if (length > sizeof(sockaddr_storage)) throw std::runtime_error("Socket address length is too large"); - len = length; - addr.reset((sockaddr*)std::malloc(len)); - std::memcpy((uint8_t*)get(), (const uint8_t*)sa, len); + set(sa, length); } SockAddr(const sockaddr_storage& ss, socklen_t len) : SockAddr((const sockaddr*)&ss, len) {} bool operator<(const SockAddr& o) const { if (len != o.len) return len < o.len; - return std::memcmp((uint8_t*)get(), (uint8_t*)o.get(), len) < 0; + return std::memcmp((const uint8_t*)get(), (const uint8_t*)o.get(), len) < 0; } bool equals(const SockAddr& o) const { return len == o.len - && std::memcmp((uint8_t*)get(), (uint8_t*)o.get(), len) == 0; + && std::memcmp((const uint8_t*)get(), (const uint8_t*)o.get(), len) == 0; } SockAddr& operator=(const SockAddr& o) { - if (len != o.len) { - len = o.len; - addr.reset((sockaddr*)std::realloc(addr.release(), len)); - } - std::memcpy((uint8_t*)get(), (const uint8_t*)o.get(), len); + set(o.get(), o.getLength()); return *this; } SockAddr& operator=(SockAddr&& o) { @@ -204,6 +193,17 @@ private: socklen_t len {0}; struct free_delete { void operator()(void* p) { std::free(p); } }; std::unique_ptr addr {}; + + void set(const sockaddr* sa, socklen_t length) { + if (len != length) { + len = length; + if (len) addr.reset((sockaddr*)std::malloc(len)); + else addr.reset(); + } + if (len) + std::memcpy((uint8_t*)get(), (const uint8_t*)sa, len); + } + }; OPENDHT_PUBLIC bool operator==(const SockAddr& a, const SockAddr& b); From 97ee4b6feae6348e5a1d80f130850fe75843e85f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Sat, 7 Oct 2017 11:35:32 +0200 Subject: [PATCH 014/156] dht: move onNewNode to RoutingTable --- include/opendht/dht.h | 1 - include/opendht/routing_table.h | 16 +++++ src/dht.cpp | 102 ++++++-------------------------- src/routing_table.cpp | 87 +++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 85 deletions(-) diff --git a/include/opendht/dht.h b/include/opendht/dht.h index 7167fa1f..48f2016a 100644 --- a/include/opendht/dht.h +++ b/include/opendht/dht.h @@ -402,7 +402,6 @@ private: Scheduler scheduler; Sp nextNodesConfirmation {}; Sp nextStorageMaintenance {}; - time_point mybucket_grow_time {time_point::min()}, mybucket6_grow_time {time_point::min()}; net::NetworkEngine network_engine; unsigned pending_pings4 {0}; diff --git a/include/opendht/routing_table.h b/include/opendht/routing_table.h index e71149b6..6873f395 100644 --- a/include/opendht/routing_table.h +++ b/include/opendht/routing_table.h @@ -24,6 +24,9 @@ namespace dht { static constexpr unsigned TARGET_NODES {8}; +namespace net { +class NetworkEngine; +} struct Bucket { Bucket() : cached() {} @@ -37,12 +40,17 @@ struct Bucket { /** Return a random node in a bucket. */ Sp randomNode(); + + void sendCachedPing(net::NetworkEngine& ne); }; class RoutingTable : public std::list { public: using std::list::list; + time_point grow_time {time_point::min()}; + bool is_client {false}; + InfoHash middle(const RoutingTable::const_iterator&) const; std::vector> findClosestNodes(const InfoHash id, time_point now, size_t count = TARGET_NODES) const; @@ -65,6 +73,14 @@ public: return empty() || (size() == 1 && front().nodes.empty()); } + void connectivityChanged(const time_point& now) { + grow_time = now; + for (auto& b : *this) + b.time = time_point::min(); + } + + bool onNewNode(const Sp& node, int comfirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne); + /** * Return a random id in the bucket's range. */ diff --git a/src/dht.cpp b/src/dht.cpp index f739280b..71569097 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -51,9 +51,9 @@ NodeStatus Dht::getStatus(sa_family_t af) const { const auto& stats = getNodesStats(af); - auto& ping = af == AF_INET ? pending_pings4 : pending_pings6; if (stats.good_nodes) return NodeStatus::Connected; + auto& ping = af == AF_INET ? pending_pings4 : pending_pings6; if (ping or stats.getKnownNodes()) return NodeStatus::Connecting; return NodeStatus::Disconnected; @@ -93,11 +93,9 @@ Dht::isRunning(sa_family_t af) const { return network_engine.isRunning(af); } const Sp Dht::findNode(const InfoHash& id, sa_family_t af) const { - const Bucket* b = findBucket(id, af); - if (!b) - return {}; - for (const auto& n : b->nodes) - if (n->id == id) return n; + if (const Bucket* b = findBucket(id, af)) + for (const auto& n : b->nodes) + if (n->id == id) return n; return {}; } @@ -105,11 +103,9 @@ Dht::findNode(const InfoHash& id, sa_family_t af) const void Dht::sendCachedPing(Bucket& b) { - if (not b.cached) - return; - DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", b.cached->toString().c_str()); - network_engine.sendPing(b.cached, nullptr, nullptr); - b.cached = {}; + if (b.cached) + DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", b.cached->toString().c_str()); + b.sendCachedPing(network_engine); } std::vector @@ -178,71 +174,8 @@ Dht::reportedAddr(const SockAddr& addr) void Dht::onNewNode(const Sp& node, int confirm) { - auto& list = buckets(node->getFamily()); - auto b = list.findBucket(node->id); - if (b == list.end()) - return; - - for (auto& n : b->nodes) { - if (n == node) { - if (confirm) - trySearchInsert(node); - return; - } - } - - /* New node. */ - /* Try adding the node to searches */ - trySearchInsert(node); - - const auto& now = scheduler.time(); - bool mybucket = list.contains(b, myid); - if (mybucket) { - if (node->getFamily() == AF_INET) - mybucket_grow_time = now; - else - mybucket6_grow_time = now; - //scheduler.edit(nextNodesConfirmation, now); - } - - if (b->nodes.size() >= TARGET_NODES) { - /* Try to get rid of an expired node. */ - for (auto& n : b->nodes) - if (n->isExpired()) { - n = node; - return; - } - /* Bucket full. Ping a dubious node */ - bool dubious = false; - for (auto& n : b->nodes) { - /* Pick the first dubious node that we haven't pinged in the - last 9 seconds. This gives nodes the time to reply, but - tends to concentrate on the same nodes, so that we get rid - of bad nodes fast. */ - if (not n->isGood(now)) { - dubious = true; - if (not n->isPendingMessage()) { - DHT_LOG.d(n->id, "[node %s] sending ping to dubious node", n->toString().c_str()); - network_engine.sendPing(n, nullptr, nullptr); - break; - } - } - } - - if ((mybucket || (is_bootstrap and list.depth(b) < 6)) && (!dubious || list.size() == 1)) { - DHT_LOG.d("Splitting from depth %u", list.depth(b)); - sendCachedPing(*b); - list.split(b); - onNewNode(node, 0); - return; - } - - /* No space for this node. Cache it away for later. */ - if (confirm or not b->cached) - b->cached = node; - } else { - /* Create a new node. */ - b->nodes.emplace_front(node); + if (buckets(node->getFamily()).onNewNode(node, confirm, scheduler.time(), myid, network_engine) or confirm) { + trySearchInsert(node); } } @@ -1393,10 +1326,7 @@ Dht::connectivityChanged(sa_family_t af) { const auto& now = scheduler.time(); scheduler.edit(nextNodesConfirmation, now); - auto& bucket_grow_time = (af == AF_INET) ? mybucket_grow_time : mybucket6_grow_time; - bucket_grow_time = now; - for (auto& b : buckets(af)) - b.time = time_point::min(); + buckets(af).connectivityChanged(now); network_engine.connectivityChanged(af); for (auto& sp : searches(af)) for (auto& sn : sp.second->nodes) { @@ -1766,10 +1696,14 @@ Dht::Dht(int s, int s6, Config config) scheduler.syncTime(); if (s < 0 && s6 < 0) return; - if (s >= 0) + if (s >= 0) { buckets4 = {Bucket {AF_INET}}; - if (s6 >= 0) + buckets4.is_client = config.is_bootstrap; + } + if (s6 >= 0) { buckets6 = {Bucket {AF_INET6}}; + buckets6.is_client = config.is_bootstrap; + } search_id = std::uniform_int_distribution{}(rd); @@ -2013,9 +1947,9 @@ Dht::confirmNodes() soon |= bucketMaintenance(buckets6); if (!soon) { - if (mybucket_grow_time >= now - seconds(150)) + if (buckets4.grow_time >= now - seconds(150)) soon |= neighbourhoodMaintenance(buckets4); - if (mybucket6_grow_time >= now - seconds(150)) + if (buckets6.grow_time >= now - seconds(150)) soon |= neighbourhoodMaintenance(buckets6); } diff --git a/src/routing_table.cpp b/src/routing_table.cpp index 495846b2..462d3c67 100644 --- a/src/routing_table.cpp +++ b/src/routing_table.cpp @@ -1,4 +1,24 @@ +/* + * Copyright (C) 2014-2017 Savoir-faire Linux Inc. + * Author(s) : Adrien Béraud + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + #include "routing_table.h" + +#include "network_engine.h" #include "rng.h" #include @@ -35,6 +55,15 @@ Bucket::randomNode() return nodes.back(); } +void Bucket::sendCachedPing(net::NetworkEngine& ne) +{ + if (not cached) + return; + //DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", cached->toString().c_str()); + ne.sendPing(cached, nullptr, nullptr); + cached = {}; +} + InfoHash RoutingTable::randomId(const RoutingTable::const_iterator& it) const { @@ -172,4 +201,62 @@ RoutingTable::split(const RoutingTable::iterator& b) return true; } +bool +RoutingTable::onNewNode(const Sp& node, int comfirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne) { + auto b = findBucket(node->id); + if (b == end()) return false; + for (auto& n : b->nodes) { + if (n == node) + return false; + } + + bool mybucket = contains(b, myid); + if (mybucket) { + grow_time = now; + //scheduler.edit(nextNodesConfirmation, now); + } + + if (b->nodes.size() >= TARGET_NODES) { + /* Try to get rid of an expired node. */ + for (auto& n : b->nodes) + if (n->isExpired()) { + n = node; + return true; + } + /* Bucket full. Ping a dubious node */ + bool dubious = false; + for (auto& n : b->nodes) { + /* Pick the first dubious node that we haven't pinged in the + last 9 seconds. This gives nodes the time to reply, but + tends to concentrate on the same nodes, so that we get rid + of bad nodes fast. */ + if (not n->isGood(now)) { + dubious = true; + if (not n->isPendingMessage()) { + //DHT_LOG.d(n->id, "[node %s] sending ping to dubious node", n->toString().c_str()); + ne.sendPing(n, nullptr, nullptr); + break; + } + } + } + + if ((mybucket || (is_client and depth(b) < 6)) && (!dubious || size() == 1)) { + //DHT_LOG.d("Splitting from depth %u", depth(b)); + b->sendCachedPing(ne); + split(b); + return onNewNode(node, comfirm, now, myid, ne); + } + + /* No space for this node. Cache it away for later. */ + if (comfirm or not b->cached) + b->cached = node; + } else { + /* Create a new node. */ + b->nodes.emplace_front(node); + } + return true; +} + + + } From a3f43885568e688a1d7a67b11658f9f818f53859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 6 Oct 2017 15:28:30 +0200 Subject: [PATCH 015/156] InfoHash: increase toFloat precision --- include/opendht/infohash.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index 57920b5b..087f7ec2 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -242,12 +242,11 @@ public: num ^= (-b ^ num) & (1 << bit); } - double - toFloat() const - { + double toFloat() const { + using D = size_t; double v = 0.; - for (unsigned i = 0; i < std::min(HASH_LEN, sizeof(unsigned)-1); i++) - v += *(cbegin()+i)/(double)(1<<(8*(i+1))); + for (size_t i = 0; i < std::min(HASH_LEN, sizeof(D)-1); i++) + v += *(cbegin()+i) / (double)((D)1 << 8*(i+1)); return v; } From d87a4ef28c1dfdeb65b4530c47ce9b5628f9e992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 6 Oct 2017 15:25:31 +0200 Subject: [PATCH 016/156] InfoHash: add operator== --- include/opendht/infohash.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index 087f7ec2..ab50d5a7 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -250,8 +250,16 @@ public: return v; } - bool - operator<(const InfoHash& o) const { + bool operator==(const InfoHash& h) const { + auto a = reinterpret_cast(data()); + auto b = reinterpret_cast(h.data()); + for (size_t i=0; i<(size()/sizeof(uint32_t)); i++) + if (a[i] != b[i]) + return false; + return true; + } + + bool operator<(const InfoHash& o) const { for(unsigned i = 0; i < HASH_LEN; i++) { if((*this)[i] != o[i]) return (*this)[i] < o[i]; From 42125367d7834a5fb2879910e27b2f990cf20669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 9 Oct 2017 14:15:26 +0200 Subject: [PATCH 017/156] SockAddr: add documentation --- include/opendht/sockaddr.h | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/include/opendht/sockaddr.h b/include/opendht/sockaddr.h index d23f6844..8f946325 100644 --- a/include/opendht/sockaddr.h +++ b/include/opendht/sockaddr.h @@ -45,6 +45,9 @@ namespace dht { OPENDHT_PUBLIC std::string print_addr(const sockaddr* sa, socklen_t slen); OPENDHT_PUBLIC std::string print_addr(const sockaddr_storage& ss, socklen_t sslen); +/** + * A Socket Address (sockaddr*), with abstraction for IPv4, IPv6 address families. + */ class OPENDHT_PUBLIC SockAddr { public: SockAddr() {} @@ -54,11 +57,18 @@ public: SockAddr(SockAddr&& o) : len(o.len), addr(std::move(o.addr)) { o.len = 0; } + + /** + * Build from existing address. + */ SockAddr(const sockaddr* sa, socklen_t length) { if (length > sizeof(sockaddr_storage)) throw std::runtime_error("Socket address length is too large"); set(sa, length); } + /** + * Build from an existing sockaddr_storage structure. + */ SockAddr(const sockaddr_storage& ss, socklen_t len) : SockAddr((const sockaddr*)&ss, len) {} bool operator<(const SockAddr& o) const { @@ -85,7 +95,17 @@ public: std::string toString() const { return print_addr(get(), getLength()); } + + /** + * Returns the address family or AF_UNSPEC if the address is not set. + */ sa_family_t getFamily() const { return len > sizeof(sa_family_t) ? addr->sa_family : AF_UNSPEC; } + + /** + * Resize the managed structure to the appropriate size (if needed), + * in which case the sockaddr structure is cleared to zero, + * and set the address family field (sa_family). + */ void setFamily(sa_family_t af) { socklen_t new_length; switch(af) { @@ -107,6 +127,10 @@ public: addr->sa_family = af; } + /** + * Retreive the port (in host byte order) or 0 if the address is not + * of a supported family. + */ in_port_t getPort() const { switch(getFamily()) { case AF_INET: @@ -117,6 +141,10 @@ public: return 0; } } + /** + * Set the port. The address must be of a supported family. + * @param p The port in host byte order. + */ void setPort(in_port_t p) { switch(getFamily()) { case AF_INET: @@ -128,12 +156,29 @@ public: } } + /** + * Returns the accessible byte length at the pointer returned by #get(). + * If zero, #get() returns null. + */ socklen_t getLength() const { return len; } + + /** + * An address is defined to be true if its length is not zero. + */ explicit operator bool() const noexcept { return len; } + /** + * Returns the address to the managed sockaddr structure. + * The accessible length is returned by #getLength(). + */ const sockaddr* get() const { return addr.get(); } + + /** + * Returns the address to the managed sockaddr structure. + * The accessible length is returned by #getLength(). + */ sockaddr* get() { return addr.get(); } const sockaddr_in& getIPv4() const { From d0bcd3934163aa067978dbcfc369565f08617fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 9 Oct 2017 15:06:34 +0200 Subject: [PATCH 018/156] IpServiceAnnouncement: use SockAddr --- include/opendht/default_types.h | 44 +++++++++++---------------------- src/default_types.cpp | 9 +++---- 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/include/opendht/default_types.h b/include/opendht/default_types.h index cbfc7d25..86582ad7 100644 --- a/include/opendht/default_types.h +++ b/include/opendht/default_types.h @@ -19,6 +19,7 @@ #pragma once #include "value.h" +#include "sockaddr.h" namespace dht { enum class ImStatus : uint8_t { @@ -204,16 +205,12 @@ private: public: static const ValueType TYPE; - IpServiceAnnouncement(in_port_t p = 0) { - ss.ss_family = 0; - setPort(p); + IpServiceAnnouncement(sa_family_t family = AF_UNSPEC, in_port_t p = 0) { + addr.setFamily(family); + addr.setPort(p); } - IpServiceAnnouncement(const sockaddr* sa, socklen_t sa_len) { - ss.ss_family = 0; - if (sa) - std::copy_n((const uint8_t*)sa, sa_len, (uint8_t*)&ss); - } + IpServiceAnnouncement(const sockaddr* sa, socklen_t sa_len) : addr(sa, sa_len) {} IpServiceAnnouncement(const Blob& b) { msgpack_unpack(unpackMsg(b).get()); @@ -222,40 +219,27 @@ public: template void msgpack_pack(Packer& pk) const { - pk.pack_array(2); - pk.pack(reinterpret_cast(&ss)->sin_port); - if (ss.ss_family == AF_INET) { - pk.pack_bin(sizeof(in_addr)); - pk.pack_bin_body((const char*)&reinterpret_cast(&ss)->sin_addr, sizeof(in_addr)); - } else if (ss.ss_family == AF_INET6) { - pk.pack_bin(sizeof(in6_addr)); - pk.pack_bin_body((const char*)&reinterpret_cast(&ss)->sin6_addr, sizeof(in6_addr)); - } + pk.pack_bin(addr.getLength()); + pk.pack_bin_body((const char*)addr.get(), addr.getLength()); } virtual void msgpack_unpack(msgpack::object o) { - if (o.type != msgpack::type::ARRAY) throw msgpack::type_error(); - if (o.via.array.size < 2) throw msgpack::type_error(); - reinterpret_cast(&ss)->sin_port = o.via.array.ptr[0].as(); - auto ip_dat = o.via.array.ptr[1].as(); - if (ip_dat.size() == sizeof(in_addr)) - std::copy(ip_dat.begin(), ip_dat.end(), (char*)&reinterpret_cast(&ss)->sin_addr); - else if (ip_dat.size() == sizeof(in6_addr)) - std::copy(ip_dat.begin(), ip_dat.end(), (char*)&reinterpret_cast(&ss)->sin6_addr); + if (o.type == msgpack::type::BIN) + addr = {(sockaddr*)o.via.bin.ptr, (socklen_t)o.via.bin.size}; else throw msgpack::type_error(); } in_port_t getPort() const { - return ntohs(reinterpret_cast(&ss)->sin_port); + return addr.getPort(); } void setPort(in_port_t p) { - reinterpret_cast(&ss)->sin_port = htons(p); + addr.setPort(p); } - sockaddr_storage getPeerAddr() const { - return ss; + const SockAddr& getPeerAddr() const { + return addr; } virtual const ValueType& getType() const { @@ -268,7 +252,7 @@ public: friend std::ostream& operator<< (std::ostream&, const IpServiceAnnouncement&); private: - sockaddr_storage ss; + SockAddr addr; }; diff --git a/src/default_types.cpp b/src/default_types.cpp index 9d59907e..7ec0e6c7 100644 --- a/src/default_types.cpp +++ b/src/default_types.cpp @@ -54,12 +54,11 @@ DhtMessage::ServiceFilter(std::string s) std::ostream& operator<< (std::ostream& s, const IpServiceAnnouncement& v) { - s << "Peer: "; - s << "port " << v.getPort(); - - if (v.ss.ss_family == AF_INET || v.ss.ss_family == AF_INET6) { + if (v.addr) { + s << "Peer: "; + s << "port " << v.getPort(); char hbuf[NI_MAXHOST]; - if (getnameinfo((sockaddr*)&v.ss, sizeof(v.ss), hbuf, sizeof(hbuf), nullptr, 0, NI_NUMERICHOST) == 0) { + if (getnameinfo(v.addr.get(), v.addr.getLength(), hbuf, sizeof(hbuf), nullptr, 0, NI_NUMERICHOST) == 0) { s << " addr " << std::string(hbuf, strlen(hbuf)); } } From 323ad774e525414a4ee232cc8779bf933223c9aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 9 Oct 2017 15:14:20 +0200 Subject: [PATCH 019/156] value: restore stream state after change --- src/value.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/value.cpp b/src/value.cpp index 101165fd..1195c172 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -35,6 +35,7 @@ Value::Filter bindFilterRaw(FilterRaw raw_filter, void* user_data) { std::ostream& operator<< (std::ostream& s, const Value& v) { + auto flags(s.flags()); s << "Value[id:" << std::hex << v.id << std::dec << " "; if (v.isEncrypted()) s << "encrypted "; @@ -63,6 +64,7 @@ std::ostream& operator<< (std::ostream& s, const Value& v) } } s << "]"; + s.flags(flags); return s; } @@ -257,9 +259,12 @@ std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi) { os << "Index["; for (auto v = fvi.index.begin(); v != fvi.index.end(); ++v) { switch (v->first) { - case Value::Field::Id: + case Value::Field::Id: { + auto flags(os.flags()); os << "Id:" << std::hex << v->second.getInt(); + os.flags(flags); break; + } case Value::Field::ValueType: os << "ValueType:" << v->second.getInt(); break; From 9f4b16e4f580f686ad62c74f873cd9024e75d909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 10 Oct 2017 17:45:34 +0200 Subject: [PATCH 020/156] cmake: add doxygen documentation support Automatically build documentation if doxygen is found. --- CMakeLists.txt | 4 ++++ doc/CMakeLists.txt | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 15b7cac2..1a7241de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,9 @@ option (OPENDHT_ARGON2 "Use included argon2 sources" OFF) option (OPENDHT_LTO "Build with LTO" OFF) option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF) +find_package(Doxygen) +option (OPENDHT_DOCUMENTATION "Create and install the HTML based API documentation (requires Doxygen)" ${DOXYGEN_FOUND}) + # Dependencies list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") find_package (Threads) @@ -90,6 +93,7 @@ set (libdir "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}") set (includedir "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") set (bindir "${CMAKE_INSTALL_FULL_BINDIR}") set (sysconfdir "${CMAKE_INSTALL_FULL_SYSCONFDIR}") +set (top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}") # Sources list (APPEND opendht_SOURCES diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 3a204fe5..681cb281 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,3 +1,16 @@ if (OPENDHT_TOOLS) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/dhtnode.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1) endif () + +if (OPENDHT_DOCUMENTATION) + if (NOT DOXYGEN_FOUND) + message(FATAL_ERROR "Doxygen is needed to build the documentation.") + endif() + configure_file (Doxyfile.in Doxyfile @ONLY) + add_custom_target(doc ALL + COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen" + VERBATIM) + install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html DESTINATION share/doc/opendht) +endif() From 73f26e120dbb4310a05275564ff8271c9bbb5246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 10 Oct 2017 17:51:11 +0200 Subject: [PATCH 021/156] doc: improve documentation --- include/opendht/crypto.h | 4 ++++ include/opendht/dht.h | 4 ++-- include/opendht/log.h | 4 ++++ include/opendht/utils.h | 6 ++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/include/opendht/crypto.h b/include/opendht/crypto.h index 07ca9199..71648288 100644 --- a/include/opendht/crypto.h +++ b/include/opendht/crypto.h @@ -36,6 +36,10 @@ extern "C" { #endif namespace dht { + +/** + * Contains all crypto primitives + */ namespace crypto { class OPENDHT_PUBLIC CryptoException : public std::runtime_error { diff --git a/include/opendht/dht.h b/include/opendht/dht.h index 48f2016a..c2403c34 100644 --- a/include/opendht/dht.h +++ b/include/opendht/dht.h @@ -168,8 +168,8 @@ public: * @param key the key for which to query data for. * @param cb a function called when new values are found on the network. * It should return false to stop the operation. - * @param donecb a function called when the operation is complete. - cb and donecb won't be called again afterward. + * @param done_cb a function called when the operation is complete. + cb and done_cb won't be called again afterward. * @param q a query used to filter values on the remotes before they send a * response. */ diff --git a/include/opendht/log.h b/include/opendht/log.h index 220eaa70..9fda9eb9 100644 --- a/include/opendht/log.h +++ b/include/opendht/log.h @@ -24,6 +24,10 @@ #include namespace dht { + +/** + * Logging-related functions + */ namespace log { /** diff --git a/include/opendht/utils.h b/include/opendht/utils.h index 054b7be4..7904d7ee 100644 --- a/include/opendht/utils.h +++ b/include/opendht/utils.h @@ -32,6 +32,9 @@ #define WANT4 1 #define WANT6 2 +/** + * OpenDHT C++ namespace + */ namespace dht { using NetId = uint32_t; @@ -93,6 +96,9 @@ public: // Serialization related definitions and utility functions +/** + * Arbitrary binary data. + */ using Blob = std::vector; /** From bdb0bef0ee192d65a5294ff9e33d2ef768c1ebbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 11 Oct 2017 02:21:50 +0200 Subject: [PATCH 022/156] network engine: move RequestAnswer out of class --- include/opendht/dht.h | 22 +++++++++---------- include/opendht/network_engine.h | 34 +++++++++++++----------------- src/dht.cpp | 36 ++++++++++++++++---------------- src/network_engine.cpp | 2 +- 4 files changed, 44 insertions(+), 50 deletions(-) diff --git a/include/opendht/dht.h b/include/opendht/dht.h index c2403c34..50215b47 100644 --- a/include/opendht/dht.h +++ b/include/opendht/dht.h @@ -504,7 +504,7 @@ private: * @param query The query sent to the node. */ void searchNodeGetDone(const net::Request& status, - net::NetworkEngine::RequestAnswer&& answer, + net::RequestAnswer&& answer, std::weak_ptr ws, Sp query); @@ -559,42 +559,42 @@ private: /* when our address is reported by a distant peer. */ void onReportedAddr(const InfoHash& id, const SockAddr&); /* when we receive a ping request */ - net::NetworkEngine::RequestAnswer onPing(Sp node); + net::RequestAnswer onPing(Sp node); /* when we receive a "find node" request */ - net::NetworkEngine::RequestAnswer onFindNode(Sp node, const InfoHash& hash, want_t want); + net::RequestAnswer onFindNode(Sp node, const InfoHash& hash, want_t want); void onFindNodeDone(const Sp& status, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp sr); /* when we receive a "get values" request */ - net::NetworkEngine::RequestAnswer onGetValues(Sp node, + net::RequestAnswer onGetValues(Sp node, const InfoHash& hash, want_t want, const Query& q); void onGetValuesDone(const Sp& status, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp& sr, const Sp& orig_query); /* when we receive a listen request */ - net::NetworkEngine::RequestAnswer onListen(Sp node, + net::RequestAnswer onListen(Sp node, const InfoHash& hash, const Blob& token, size_t socket_id, const Query& query); void onListenDone(const Sp& status, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp& sr); /* when we receive an announce request */ - net::NetworkEngine::RequestAnswer onAnnounce(Sp node, + net::RequestAnswer onAnnounce(Sp node, const InfoHash& hash, const Blob& token, const std::vector>& v, const time_point& created); - net::NetworkEngine::RequestAnswer onRefresh(Sp node, + net::RequestAnswer onRefresh(Sp node, const InfoHash& hash, const Blob& token, const Value::Id& vid); void onAnnounceDone(const Sp& status, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp& sr); }; diff --git a/include/opendht/network_engine.h b/include/opendht/network_engine.h index dee6584b..a46b5918 100644 --- a/include/opendht/network_engine.h +++ b/include/opendht/network_engine.h @@ -81,6 +81,20 @@ private: struct ParsedMessage; +/** + * Answer for a request. + */ +struct RequestAnswer { + Blob ntoken {}; + Value::Id vid {}; + std::vector> values {}; + std::vector> fields {}; + std::vector> nodes4 {}; + std::vector> nodes6 {}; + RequestAnswer() {} + RequestAnswer(ParsedMessage&& msg); +}; + /*! * @class NetworkEngine * @brief An abstraction of communication protocol on the network. @@ -101,26 +115,6 @@ struct ParsedMessage; */ class NetworkEngine final { -public: - /*! - * @class RequestAnswer - * @brief Answer for a request. - * @details - * Answer for a request to be (de)serialized. Used for reponding to a node - * and looking up the response from a node. - */ - struct RequestAnswer { - Blob ntoken {}; - Value::Id vid {}; - std::vector> values {}; - std::vector> fields {}; - std::vector> nodes4 {}; - std::vector> nodes6 {}; - RequestAnswer() {} - RequestAnswer(ParsedMessage&& msg); - }; - - private: /** * Called when we receive an error message. diff --git a/src/dht.cpp b/src/dht.cpp index 71569097..878625e3 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -218,7 +218,7 @@ Dht::expireSearches() void Dht::searchNodeGetDone(const net::Request& req, - net::NetworkEngine::RequestAnswer&& answer, + net::RequestAnswer&& answer, std::weak_ptr ws, Sp query) { @@ -262,7 +262,7 @@ void Dht::paginate(std::weak_ptr ws, Sp query, SearchNode* n) { if (not sr) return; auto select_q = std::make_shared(Select {}.field(Value::Field::Id), query ? query->where : Where {}); auto onSelectDone = [this,ws,query](const net::Request& status, - net::NetworkEngine::RequestAnswer&& answer) mutable { + net::RequestAnswer&& answer) mutable { // retreive search auto sr = ws.lock(); if (not sr) return; @@ -393,7 +393,7 @@ void Dht::searchSendAnnounceValue(const Sp& sr) { if (not something_to_announce) continue; - auto onDone = [this,ws](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer) + auto onDone = [this,ws](const net::Request& req, net::RequestAnswer&& answer) { /* when put done */ if (auto sr = ws.lock()) { onAnnounceDone(req.node, answer, sr); @@ -407,7 +407,7 @@ void Dht::searchSendAnnounceValue(const Sp& sr) { scheduler.edit(sr->nextSearchStep, scheduler.time()); }; auto onSelectDone = - [this,ws,onDone,onExpired](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer) mutable + [this,ws,onDone,onExpired](const net::Request& req, net::RequestAnswer&& answer) mutable { /* on probing done */ const auto& now = scheduler.time(); if (auto sr = ws.lock()) { @@ -549,7 +549,7 @@ Dht::searchStep(Sp sr) std::weak_ptr ws = sr; n.listenStatus[query] = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req, - [this,ws,query](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer) mutable + [this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable { /* on done */ if (auto sr = ws.lock()) { onListenDone(req.node, answer, sr); @@ -565,7 +565,7 @@ Dht::searchStep(Sp sr) sn->listenStatus.erase(query); } }, - [this,ws,query](const Sp& node, net::NetworkEngine::RequestAnswer&& answer) mutable + [this,ws,query](const Sp& node, net::RequestAnswer&& answer) mutable { /* on new values */ if (auto sr = ws.lock()) { onGetValuesDone(node, answer, sr, query); @@ -2075,7 +2075,7 @@ Dht::pingNode(const sockaddr* sa, socklen_t salen, DoneCallbackSimple&& cb) DHT_LOG.d("Sending ping to %s", print_addr(sa, salen).c_str()); auto& count = sa->sa_family == AF_INET ? pending_pings4 : pending_pings6; count++; - network_engine.sendPing(sa, salen, [&count,cb](const net::Request&, net::NetworkEngine::RequestAnswer&&) { + network_engine.sendPing(sa, salen, [&count,cb](const net::Request&, net::RequestAnswer&&) { count--; if (cb) cb(true); @@ -2119,17 +2119,17 @@ Dht::onReportedAddr(const InfoHash& id, const SockAddr& addr) reportedAddr(addr); } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onPing(Sp) { return {}; } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onFindNode(Sp node, const InfoHash& target, want_t want) { const auto& now = scheduler.time(); - net::NetworkEngine::RequestAnswer answer; + net::RequestAnswer answer; answer.ntoken = makeToken(node->addr, false); if (want & WANT4) answer.nodes4 = buckets4.findClosestNodes(target, now, TARGET_NODES); @@ -2138,7 +2138,7 @@ Dht::onFindNode(Sp node, const InfoHash& target, want_t want) return answer; } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onGetValues(Sp node, const InfoHash& hash, want_t, const Query& query) { if (hash == zeroes) { @@ -2149,7 +2149,7 @@ Dht::onGetValues(Sp node, const InfoHash& hash, want_t, const Query& query }; } const auto& now = scheduler.time(); - net::NetworkEngine::RequestAnswer answer {}; + net::RequestAnswer answer {}; auto st = store.find(hash); answer.ntoken = makeToken(node->addr, false); answer.nodes4 = buckets4.findClosestNodes(hash, now, TARGET_NODES); @@ -2164,7 +2164,7 @@ Dht::onGetValues(Sp node, const InfoHash& hash, want_t, const Query& query } void Dht::onGetValuesDone(const Sp& node, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp& sr, const Sp& orig_query) { @@ -2235,7 +2235,7 @@ void Dht::onGetValuesDone(const Sp& node, } } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onListen(Sp node, const InfoHash& hash, const Blob& token, size_t socket_id, const Query& query) { if (hash == zeroes) { @@ -2256,7 +2256,7 @@ Dht::onListen(Sp node, const InfoHash& hash, const Blob& token, size_t soc void Dht::onListenDone(const Sp& node, - net::NetworkEngine::RequestAnswer& answer, + net::RequestAnswer& answer, Sp& sr) { DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got listen confirmation", @@ -2269,7 +2269,7 @@ Dht::onListenDone(const Sp& node, } } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onAnnounce(Sp node, const InfoHash& hash, const Blob& token, @@ -2337,7 +2337,7 @@ Dht::onAnnounce(Sp node, return {}; } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onRefresh(Sp node, const InfoHash& hash, const Blob& token, const Value::Id& vid) { using namespace net; @@ -2360,7 +2360,7 @@ Dht::onRefresh(Sp node, const InfoHash& hash, const Blob& token, const Val } void -Dht::onAnnounceDone(const Sp& node, net::NetworkEngine::RequestAnswer& answer, Sp& sr) +Dht::onAnnounceDone(const Sp& node, net::RequestAnswer& answer, Sp& sr) { DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got reply to put!", sr->id.toString().c_str(), node->toString().c_str()); diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 8f23c361..ca50133c 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -169,7 +169,7 @@ serializeValues(const std::vector>& st) return svals; } -NetworkEngine::RequestAnswer::RequestAnswer(ParsedMessage&& msg) +RequestAnswer::RequestAnswer(ParsedMessage&& msg) : ntoken(std::move(msg.token)), values(std::move(msg.values)), fields(std::move(msg.fields)), nodes4(std::move(msg.nodes4)), nodes6(std::move(msg.nodes6)) {} From be7494961cabdb6655015f577e7c5b4233366332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 11 Oct 2017 02:23:56 +0200 Subject: [PATCH 023/156] network engine: move socket management to Node --- include/opendht/network_engine.h | 19 ---------- include/opendht/node.h | 28 ++++++++++++++ src/network_engine.cpp | 63 ++++++++------------------------ src/node.cpp | 29 +++++++++++++++ src/request.h | 22 +++++------ 5 files changed, 83 insertions(+), 78 deletions(-) diff --git a/include/opendht/network_engine.h b/include/opendht/network_engine.h index a46b5918..0dde4a6b 100644 --- a/include/opendht/network_engine.h +++ b/include/opendht/network_engine.h @@ -200,7 +200,6 @@ private: const Value::Id&)> onRefresh {}; public: - using SocketCb = std::function&, RequestAnswer&&)>; using RequestCb = std::function; using RequestExpiredCb = std::function; @@ -386,23 +385,6 @@ public: const Blob& token, RequestCb&& on_done, RequestExpiredCb&& on_expired); - /** - * Opens a socket on which a node will be able allowed to write for further - * additionnal updates following the response to a previous request. - * - * @param node The node which will be allowed to write on this socket. - * @param cb The callback to execute once updates arrive on the socket. - * - * @return the socket. - */ - Sp openSocket(const Sp& node, TransPrefix tp, SocketCb&& cb); - - /** - * Closes a socket so that no further data will be red on that socket. - * - * @param socket The socket to close. - */ - void closeSocket(Sp socket); /** * Parses a message and calls appropriate callbacks. @@ -557,7 +539,6 @@ private: uint16_t transaction_id {1}; std::map> requests {}; std::map partial_messages; - std::map> opened_sockets {}; MessageStats in_stats {}, out_stats {}; std::set blacklist {}; diff --git a/include/opendht/node.h b/include/opendht/node.h index 15791a41..a36b4c51 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -22,13 +22,19 @@ #include "infohash.h" // includes socket structures #include "utils.h" #include "sockaddr.h" +#include "net.h" #include +#include namespace dht { +struct Node; namespace net { struct Request; +struct Socket; +struct RequestAnswer; +using SocketCb = std::function&, RequestAnswer&&)>; } /* namespace net */ struct Node { @@ -84,6 +90,26 @@ struct Node { void setExpired(); + /** + * Opens a socket on which a node will be able allowed to write for further + * additionnal updates following the response to a previous request. + * + * @param node The node which will be allowed to write on this socket. + * @param cb The callback to execute once updates arrive on the socket. + * + * @return the socket. + */ + Sp openSocket(const net::TransId& id, net::SocketCb&& cb); + + Sp getSocket(const net::TransId& id) const; + + /** + * Closes a socket so that no further data will be red on that socket. + * + * @param socket The socket to close. + */ + void closeSocket(const Sp& socket); + /** * Resets the state of the node so it's not expired anymore. */ @@ -106,6 +132,8 @@ private: static const constexpr unsigned MAX_AUTH_ERRORS {3}; std::list> requests_ {}; + std::map> sockets_ {}; + unsigned auth_errors {0}; bool expired_ {false}; diff --git a/src/network_engine.cpp b/src/network_engine.cpp index ca50133c..73991902 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -232,29 +232,12 @@ NetworkEngine::isRunning(sa_family_t af) const } } -Sp -NetworkEngine::openSocket(const Sp& node, TransPrefix tp, SocketCb&& cb) -{ - auto tid = TransId {tp, getNewTid()}; - auto s = opened_sockets.emplace(tid, std::make_shared(node, tid, cb)); - if (not s.second) - DHT_LOG.e(node->id, "[node %s] socket (tid: %d) already opened!", node->id.toString().c_str(), tid.toInt()); - return s.first->second; -} - -void -NetworkEngine::closeSocket(Sp socket) -{ - if (socket) - opened_sockets.erase(socket->id); -} - void NetworkEngine::cancelRequest(Sp& req) { if (req) { req->cancel(); - closeSocket(req->socket); + req->node->closeSocket(req->socket); requests.erase(req->tid); } } @@ -503,30 +486,23 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro const auto& now = scheduler.time(); if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { + auto node = cache.getNode(msg->id, from, now, true); + auto rsocket = node->getSocket(msg->tid); + //auto request = node->getRequest(msg->tid); + /* either response for a request or data for an opened socket */ auto req_it = requests.find(msg->tid); - auto rsocket_it = opened_sockets.end(); - if (req_it == requests.end()) - rsocket_it = opened_sockets.find(msg->tid); - if (req_it == requests.end() and rsocket_it == opened_sockets.end()) + if (req_it == requests.end() and not rsocket) { throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find transaction", msg->id}; + } auto req = req_it != requests.end() ? req_it->second : nullptr; - auto rsocket = rsocket_it != opened_sockets.end() ? rsocket_it->second : nullptr; - - auto& node = req ? req->node : rsocket->node; - if (node->id != msg->id) { - if (node->id == zeroes) // received reply to a message sent when we didn't know the node ID. - node = cache.getNode(msg->id, from, now, true); - else { - // received reply from unexpected node - node->received(now, req); - onNewNode(node, 2); - DHT_LOG.w(node->id, "[node %s] message received from unexpected node", node->toString().c_str()); - return; - } - } else - node->update(from); + if (req and req->node != node and req->node->id != zeroes) { + node->received(now, req); + onNewNode(node, 2); + DHT_LOG.w(node->id, "[node %s] message received from unexpected node", node->toString().c_str()); + return; + } node->received(now, req); @@ -569,7 +545,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro break; } else { /* request socket data */ deserializeNodes(*msg, from); - rsocket->on_receive(rsocket->node, std::move(*msg)); + rsocket->on_receive(node, std::move(*msg)); } default: break; @@ -624,9 +600,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro case MessageType::Listen: { DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str()); ++in_stats.listen; - /* TODO: backward compatibility check to remove in a few versions (see ::sendListen doc) */ - auto socket_id = msg->socket_id.getTid() ? msg->socket_id : TransId { TransPrefix::GET_VALUES, msg->tid.getTid() }; - RequestAnswer answer = onListen(node, msg->info_hash, msg->token, socket_id.toInt(), std::move(msg->query)); + RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id.toInt(), std::move(msg->query)); sendListenConfirmation(from, msg->tid); break; } @@ -1074,12 +1048,7 @@ NetworkEngine::sendListen(Sp n, } else { if (previous) DHT_LOG.e(hash, "[node %s] trying refresh listen contract with wrong node", previous->node->toString().c_str()); - /* TODO: Manually creating a socket for "listen" backward compatibility. - * As soon as this is not an issue anymore, switch to using - * ::createSocket function! */ - auto sid = TransId {TransPrefix::GET_VALUES, tid.getTid()}; - socket = std::make_shared(n, sid, socket_cb); - opened_sockets.emplace(sid, socket); + socket = n->openSocket(TransId {TransPrefix::GET_VALUES, getNewTid()}, std::move(socket_cb)); } if (not socket) { diff --git a/src/node.cpp b/src/node.cpp index 1c482af2..f4c3d0c8 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -104,9 +104,38 @@ Node::setExpired() r->setExpired(); } requests_.clear(); + sockets_.clear(); } +Sp +Node::openSocket(const net::TransId& tid, net::SocketCb&& cb) +{ + auto s = sockets_.emplace(tid, std::make_shared(tid, cb)); + //if (not s.second) + // DHT_LOG.e(id, "[node %s] socket (tid: %d) already opened!", id.toString().c_str(), tid.toInt()); + //else + // DHT_LOG.w("Opened socket (tid: %d), %lu opened", s.first->second->id, sockets_.size()); + return s.first->second; +} + + +Sp +Node::getSocket(const net::TransId& tid) const +{ + auto it = sockets_.find(tid); + return it == sockets_.end() ? nullptr : it->second; +} + +void +Node::closeSocket(const Sp& socket) +{ + if (socket) { + sockets_.erase(socket->id); + //DHT_LOG.w("Closing socket (tid: %d), %lu remaining", socket->id, sockets_.size()); + } +} + std::string Node::toString() const { diff --git a/src/request.h b/src/request.h index 0174e3b0..b377e610 100644 --- a/src/request.h +++ b/src/request.h @@ -20,17 +20,19 @@ #pragma once #include "net.h" +#include "value.h" namespace dht { +struct Node; namespace net { class NetworkEngine; struct ParsedMessage; +struct RequestAnswer; +using SocketCb = std::function&, RequestAnswer&&)>; -/*! - * @class Socket - * @brief Open route to a node for continous incoming packets. - * @details +/** + * Open route to a node for continous incoming packets. * A socket lets a remote node send us continuous packets treated using a * given callback. This is intended to provide an easy management of * specific updates nodes can send. For e.g, this is used in the case of the @@ -38,14 +40,10 @@ struct ParsedMessage; */ struct Socket { Socket() {} - Socket(std::shared_ptr node, - TransId id, - std::function&, ParsedMessage&&)> on_receive) : - node(node), id(id), on_receive(on_receive) {} - - std::shared_ptr node; + Socket(TransId id, SocketCb on_receive) : + id(id), on_receive(on_receive) {} TransId id; - std::function&, ParsedMessage&&)> on_receive {}; + SocketCb on_receive {}; }; /*! @@ -59,7 +57,7 @@ struct Socket { struct Request { friend class dht::net::NetworkEngine; - std::shared_ptr node {}; /* the node to whom the request is destined. */ + Sp node {}; /* the node to whom the request is destined. */ time_point reply_time {time_point::min()}; /* time when we received the response to the request. */ enum class State From 8c529b7d7fd21871bb743cdd24cfbf56d04b3cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 11 Oct 2017 13:47:53 +0200 Subject: [PATCH 024/156] request: add getTid(), getSocket() --- src/request.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/request.h b/src/request.h index b377e610..4604059f 100644 --- a/src/request.h +++ b/src/request.h @@ -77,13 +77,17 @@ struct Request { Request(State state = State::PENDING) : state_(state) {} Request(TransId tid, - std::shared_ptr node, + Sp node, Blob&& msg, std::function on_done, std::function on_expired, - std::shared_ptr socket = {}) : + Sp socket = {}) : node(node), on_done(on_done), on_expired(on_expired), tid(tid), msg(std::move(msg)), socket(socket) { } + TransId getTid() const { return tid; } + + const Sp& getSocket() const { return socket; } + void setExpired() { if (pending()) { state_ = Request::State::EXPIRED; @@ -130,7 +134,7 @@ private: const TransId tid; /* the request id. */ Blob msg {}; /* the serialized message. */ - std::shared_ptr socket; /* the socket used for further reponses. */ + Sp socket; /* the socket used for further reponses. */ }; } /* namespace net */ From 3bb5e7ee12da3a87d0b0473f13830299d221441c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 11 Oct 2017 17:36:10 +0200 Subject: [PATCH 025/156] network engine: move transaction ownership to Node --- include/opendht/network_engine.h | 8 +-- include/opendht/node.h | 14 ++--- src/dht.cpp | 51 +++++----------- src/network_engine.cpp | 101 +++++++++++-------------------- src/node.cpp | 58 +++++++++++------- src/search.h | 62 +++++++++++++++++-- 6 files changed, 148 insertions(+), 146 deletions(-) diff --git a/include/opendht/network_engine.h b/include/opendht/network_engine.h index 0dde4a6b..84dac9db 100644 --- a/include/opendht/network_engine.h +++ b/include/opendht/network_engine.h @@ -239,12 +239,6 @@ public: bool isRunning(sa_family_t af) const; inline want_t want () const { return dht_socket >= 0 && dht_socket6 >= 0 ? (WANT4 | WANT6) : -1; } - /** - * Cancel a request. Setting req->cancelled = true is not enough in the case - * a request is "persistent". - */ - void cancelRequest(Sp& req); - void connectivityChanged(sa_family_t); /************** @@ -458,7 +452,7 @@ private: * Sends a request to a node. Request::MAX_ATTEMPT_COUNT attempts will * be made before the request expires. */ - void sendRequest(Sp& request); + void sendRequest(const Sp& request); /** * Generates a new request id, skipping the invalid id. diff --git a/include/opendht/node.h b/include/opendht/node.h index a36b4c51..6afdcd61 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -85,8 +85,10 @@ struct Node { void update(const SockAddr&); - void requested(std::shared_ptr& req); - void received(time_point now, std::shared_ptr req); + void requested(const Sp& req); + void received(time_point now, const Sp& req); + Sp getRequest(const net::TransId& tid); + void cancelRequest(const Sp& req); void setExpired(); @@ -131,17 +133,11 @@ private: /* Number of times we accept authentication errors from this node. */ static const constexpr unsigned MAX_AUTH_ERRORS {3}; - std::list> requests_ {}; + std::map> requests_ {}; std::map> sockets_ {}; unsigned auth_errors {0}; bool expired_ {false}; - - void clearPendingQueue() { - requests_.remove_if([](std::weak_ptr& w) { - return w.expired(); - }); - } }; } diff --git a/src/dht.cpp b/src/dht.cpp index 878625e3..35a8ff19 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -866,35 +866,13 @@ Dht::cancelListen(const InfoHash& id, size_t token) if (st != store.end() && tokenlocal) st->second.local_listeners.erase(tokenlocal); - auto searches_cancel_listen = [&](std::map> srs) { - for (auto& sp : srs) { - auto& s = sp.second; - if (s->id != id) continue; - auto af_token = s->af == AF_INET ? std::get<1>(it->second) : std::get<2>(it->second); - if (af_token == 0) - continue; - Sp query; - const auto& ll = s->listeners.find(af_token); - if (ll != s->listeners.cend()) - query = ll->second.query; - for (auto& sn : s->nodes) { - if (s->listeners.empty()) { /* also erase requests for all searchnodes. */ - for (auto& ls : sn.listenStatus) - network_engine.cancelRequest(ls.second); - sn.listenStatus.clear(); - } else if (query) { - auto it = sn.listenStatus.find(query); - if (it != sn.listenStatus.end()) { - network_engine.cancelRequest(it->second); - sn.listenStatus.erase(it); - } - } - } - s->listeners.erase(af_token); - } + auto searches_cancel_listen = [&id](std::map>& srs, size_t token) { + auto srp = srs.find(id); + if (srp != srs.end() and token) + srp->second->cancelListen(token); }; - searches_cancel_listen(searches4); - searches_cancel_listen(searches6); + searches_cancel_listen(searches4, std::get<1>(it->second)); + searches_cancel_listen(searches6, std::get<2>(it->second)); listeners.erase(it); return true; } @@ -1331,7 +1309,7 @@ Dht::connectivityChanged(sa_family_t af) for (auto& sp : searches(af)) for (auto& sn : sp.second->nodes) { for (auto& ls : sn.listenStatus) - network_engine.cancelRequest(ls.second); + sn.node->cancelRequest(ls.second); sn.listenStatus.clear(); } reported_addr.erase(std::remove_if(reported_addr.begin(), reported_addr.end(), [&](const ReportedAddr& addr){ @@ -2090,14 +2068,15 @@ Dht::pingNode(const sockaddr* sa, socklen_t salen, DoneCallbackSimple&& cb) void Dht::onError(Sp req, net::DhtProtocolException e) { + const auto& node = req->node; if (e.getCode() == net::DhtProtocolException::UNAUTHORIZED) { - DHT_LOG.e(req->node->id, "[node %s] token flush", req->node->toString().c_str()); - req->node->authError(); - network_engine.cancelRequest(req); - for (auto& srp : searches(req->node->getFamily())) { + DHT_LOG.e(node->id, "[node %s] token flush", node->toString().c_str()); + node->authError(); + node->cancelRequest(req); + for (auto& srp : searches(node->getFamily())) { auto& sr = srp.second; for (auto& n : sr->nodes) { - if (n.node != req->node) continue; + if (n.node != node) continue; n.token.clear(); n.last_get_reply = time_point::min(); searchSendGetValues(sr); @@ -2105,8 +2084,8 @@ Dht::onError(Sp req, net::DhtProtocolException e) { } } } else if (e.getCode() == net::DhtProtocolException::NOT_FOUND) { - DHT_LOG.e(req->node->id, "[node %s] returned error 404: storage not found", req->node->toString().c_str()); - network_engine.cancelRequest(req); + DHT_LOG.e(node->id, "[node %s] returned error 404: storage not found", node->toString().c_str()); + node->cancelRequest(req); } } diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 73991902..080fa737 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -232,23 +232,9 @@ NetworkEngine::isRunning(sa_family_t af) const } } -void -NetworkEngine::cancelRequest(Sp& req) -{ - if (req) { - req->cancel(); - req->node->closeSocket(req->socket); - requests.erase(req->tid); - } -} - void NetworkEngine::clear() -{ - for (auto& req : requests) - req.second->setExpired(); - requests.clear(); -} +{} void NetworkEngine::connectivityChanged(sa_family_t af) @@ -260,18 +246,16 @@ void NetworkEngine::requestStep(Sp sreq) { auto& req = *sreq; - if (not req.pending()) { - if (req.cancelled()) - requests.erase(req.tid); + if (not req.pending()) return; - } auto now = scheduler.time(); auto& node = *req.node; if (req.isExpired(now)) { DHT_LOG.e(node.id, "[node %s] expired !", node.toString().c_str()); node.setExpired(); - requests.erase(req.tid); + if (node.id == zeroes) + requests.erase(req.tid); return; } else if (req.attempt_count == 1) { req.on_expired(req, false); @@ -285,7 +269,8 @@ NetworkEngine::requestStep(Sp sreq) err == EAFNOSUPPORT) { node.setExpired(); - requests.erase(req.tid); + if (node.id == zeroes) + requests.erase(req.tid); } else { if (err != EAGAIN) { ++req.attempt_count; @@ -304,18 +289,13 @@ NetworkEngine::requestStep(Sp sreq) * be made before the request expires. */ void -NetworkEngine::sendRequest(Sp& request) +NetworkEngine::sendRequest(const Sp& request) { + auto& node = request->node; + if (node->id == zeroes) + requests.emplace(request->tid, request); request->start = scheduler.time(); - auto e = requests.emplace(request->tid, request); - if (not e.second) { - // Should not happen ! - // Try to handle this scenario as well as we can - e.first->second->setExpired(); - e.first->second = request; - DHT_LOG.e(request->node->id, "Request already existed (tid: %d)!", request->tid.getTid()); - } - request->node->requested(request); + node->requested(request); requestStep(request); } @@ -375,14 +355,6 @@ void NetworkEngine::blacklistNode(const Sp& n) { n->setExpired(); - for (auto rit = requests.begin(); rit != requests.end();) { - if (rit->second->node == n) { - //rit->second->cancel(); - requests.erase(rit++); - } else { - ++rit; - } - } blacklist.emplace(n->addr); } @@ -488,20 +460,20 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { auto node = cache.getNode(msg->id, from, now, true); auto rsocket = node->getSocket(msg->tid); - //auto request = node->getRequest(msg->tid); + auto req = node->getRequest(msg->tid); /* either response for a request or data for an opened socket */ - auto req_it = requests.find(msg->tid); - if (req_it == requests.end() and not rsocket) { - throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find transaction", msg->id}; - } - - auto req = req_it != requests.end() ? req_it->second : nullptr; - if (req and req->node != node and req->node->id != zeroes) { - node->received(now, req); - onNewNode(node, 2); - DHT_LOG.w(node->id, "[node %s] message received from unexpected node", node->toString().c_str()); - return; + if (not req and not rsocket) { + auto req_it = requests.find(msg->tid); + if (req_it != requests.end() and req_it->second->node->id == zeroes) { + req = req_it->second; + req->node = node; + requests.erase(req_it); + } else { + node->received(now, req); + onNewNode(node, 1); + throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find transaction", msg->id}; + } } node->received(now, req); @@ -511,7 +483,6 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro if (req and (req->cancelled() or req->expired() or req->completed())) { DHT_LOG.w(node->id, "[node %s] response to expired, cancelled or completed request", node->toString().c_str()); - requests.erase(req_it); return; } @@ -532,21 +503,19 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro break; } case MessageType::Reply: - if (req) { /* request reply */ - if (msg->type == MessageType::AnnounceValue or msg->type == MessageType::Listen) - req->node->authSuccess(); + if (req) { /* request reply */ + if (msg->type == MessageType::AnnounceValue or msg->type == MessageType::Listen) + req->node->authSuccess(); + req->reply_time = scheduler.time(); - // erase before calling callback to make sure iterator is still valid - requests.erase(req_it); - req->reply_time = scheduler.time(); - - deserializeNodes(*msg, from); - req->setDone(std::move(*msg)); - break; - } else { /* request socket data */ - deserializeNodes(*msg, from); - rsocket->on_receive(node, std::move(*msg)); - } + deserializeNodes(*msg, from); + req->setDone(std::move(*msg)); + break; + } else { /* request socket data */ + deserializeNodes(*msg, from); + rsocket->on_receive(node, std::move(*msg)); + } + break; default: break; } diff --git a/src/node.cpp b/src/node.cpp index f4c3d0c8..a6bcc77e 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -41,11 +41,9 @@ Node::isGood(time_point now) const bool Node::isPendingMessage() const { - for (auto w : requests_) { - if (auto r = w.lock()) { - if (r->pending()) - return true; - } + for (auto r : requests_) { + if (r.second->pending()) + return true; } return false; } @@ -54,11 +52,9 @@ size_t Node::getPendingMessageCount() const { size_t count {0}; - for (auto w : requests_) { - if (auto r = w.lock()) { - if (r->pending()) - count++; - } + for (auto r : requests_) { + if (r.second->pending()) + count++; } return count; } @@ -71,27 +67,44 @@ Node::update(const SockAddr& new_addr) /** To be called when a message was sent to the node */ void -Node::requested(std::shared_ptr& req) +Node::requested(const Sp& req) { - requests_.emplace_back(req); + auto e = requests_.emplace(req->getTid(), req); + if (not e.second) { + // Should not happen ! + // Try to handle this scenario as well as we can + e.first->second->setExpired(); + e.first->second = req; + } } /** To be called when a message was received from the node. Req should be true if the message was an aswer to a request we made*/ void -Node::received(time_point now, std::shared_ptr req) +Node::received(time_point now, const Sp& req) { time = now; expired_ = false; if (req) { reply_time = now; - for (auto it = requests_.begin(); it != requests_.end();) { - auto r = it->lock(); - if (not r or r == req) - it = requests_.erase(it); - else - ++it; - } + requests_.erase(req->getTid()); + } +} + +Sp +Node::getRequest(const net::TransId& tid) +{ + auto it = requests_.find(tid); + return it != requests_.end() ? it->second : nullptr; +} + +void +Node::cancelRequest(const Sp& req) +{ + if (req) { + req->cancel(); + closeSocket(req->getSocket()); + requests_.erase(req->getTid()); } } @@ -99,9 +112,8 @@ void Node::setExpired() { expired_ = true; - for (auto w : requests_) { - if (auto r = w.lock()) - r->setExpired(); + for (auto r : requests_) { + r.second->setExpired(); } requests_.clear(); sockets_.clear(); diff --git a/src/search.h b/src/search.h index 81a3fe94..9d7ae861 100644 --- a/src/search.h +++ b/src/search.h @@ -77,6 +77,13 @@ struct Dht::SearchNode { SearchNode() : node() {} SearchNode(const Sp& node) : node(node) {} + ~SearchNode() { + if (node) { + cancelGet(); + cancelListen(); + cancelAnnounce(); + } + } /** * Can we use this node to listen/announce now ? @@ -173,6 +180,14 @@ struct Dht::SearchNode { } } + void cancelGet() { + for (const auto& status : getStatus) { + if (status.second->pending()) { + node->cancelRequest(status.second); + } + } + } + /** * Tells if a request in the status map is expired. * @@ -180,12 +195,12 @@ struct Dht::SearchNode { * * @return true if there exists an expired request, else false. */ - bool expired(const SyncStatus& status) const { + /*static bool expired(const SyncStatus& status) const { return std::find_if(status.begin(), status.end(), [](const SyncStatus::value_type& r){ return r.second and r.second->expired(); }) != status.end(); - } + }*/ /** * Tells if a request in the status map is pending. @@ -194,19 +209,29 @@ struct Dht::SearchNode { * * @return true if there exists an expired request, else false. */ - bool pending(const SyncStatus& status) const { + static bool pending(const SyncStatus& status) { return std::find_if(status.begin(), status.end(), [](const SyncStatus::value_type& r){ return r.second and r.second->pending(); }) != status.end(); } + bool pendingGet() const { return pending(getStatus); } + bool isAnnounced(Value::Id vid) const { auto ack = acked.find(vid); if (ack == acked.end() or not ack->second.first) return false; return ack->second.first->completed(); } + void cancelAnnounce() { + for (const auto& status : acked) { + const auto& req = status.second.first; + if (req->pending()) { + node->cancelRequest(req); + } + } + } bool isListening(time_point now) const { auto ls = listenStatus.begin(); @@ -229,6 +254,18 @@ struct Dht::SearchNode { return false; return listen_status->second->reply_time + LISTEN_EXPIRE_TIME > now; } + void cancelListen() { + for (const auto& status : listenStatus) + node->cancelRequest(status.second); + listenStatus.clear(); + } + void cancelListen(const Sp& query) { + auto it = listenStatus.find(query); + if (it != listenStatus.end()) { + node->cancelRequest(it->second); + listenStatus.erase(it); + } + } /** * Assumng the node is synced, should a "put" request be sent to this node now ? @@ -319,7 +356,7 @@ struct Dht::Search { unsigned currentlySolicitedNodeCount() const { unsigned count = 0; for (const auto& n : nodes) - if (not n.isBad() and n.pending(n.getStatus)) + if (not n.isBad() and n.pendingGet()) count++; return count; } @@ -382,6 +419,21 @@ struct Dht::Search { bool isAnnounced(Value::Id id) const; bool isListening(time_point now) const; + void cancelListen(size_t token) { + Sp query; + const auto& ll = listeners.find(token); + if (ll != listeners.cend()) { + query = ll->second.query; + listeners.erase(ll); + } + for (auto& sn : nodes) { + if (listeners.empty()) + sn.cancelListen(); + else if (query) + sn.cancelListen(query); + } + } + /** * @return The number of non-good search nodes. */ @@ -672,7 +724,7 @@ Dht::Search::getUpdateTime(time_point now) const for (const auto& sn : nodes) { if (sn.node->isExpired() or (sn.candidate and t >= TARGET_NODES)) continue; - auto pending = sn.pending(sn.getStatus); + auto pending = sn.pendingGet(); if (sn.last_get_reply < std::max(now - Node::NODE_EXPIRE_TIME, last_get) or pending) { // not isSynced if (not pending and solicited_nodes < MAX_REQUESTED_SEARCH_NODES) From 98ec789526691f094a754c09d710e2dd332a936c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 11 Oct 2017 16:51:42 +0200 Subject: [PATCH 026/156] nertwork engine: move tid management to Node --- include/opendht/network_engine.h | 11 ----------- include/opendht/node.h | 16 ++++++++++++++-- src/network_engine.cpp | 15 +++++++-------- src/node.cpp | 10 ++++++++++ src/request.h | 2 +- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/include/opendht/network_engine.h b/include/opendht/network_engine.h index 84dac9db..2e309927 100644 --- a/include/opendht/network_engine.h +++ b/include/opendht/network_engine.h @@ -454,16 +454,6 @@ private: */ void sendRequest(const Sp& request); - /** - * Generates a new request id, skipping the invalid id. - * - * @return the new id. - */ - uint16_t getNewTid() { - ++transaction_id; - return transaction_id == TransId::INVALID ? ++transaction_id : transaction_id; - } - struct MessageStats { unsigned ping {0}; unsigned find {0}; @@ -530,7 +520,6 @@ private: RateLimiter rate_limiter {}; // requests handling - uint16_t transaction_id {1}; std::map> requests {}; std::map partial_messages; diff --git a/include/opendht/node.h b/include/opendht/node.h index 6afdcd61..fa00d4dd 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -44,9 +44,9 @@ struct Node { time_point time {time_point::min()}; /* last time eared about */ time_point reply_time {time_point::min()}; /* time of last correct reply received */ + Node(const InfoHash& id, const SockAddr& addr); Node(const InfoHash& id, const sockaddr* sa, socklen_t salen) - : id(id), addr(sa, salen) {} - Node(const InfoHash& id, const SockAddr& addr) : id(id), addr(addr) {} + : Node(id, SockAddr(sa, salen)) {} InfoHash getId() const { return id; @@ -117,6 +117,16 @@ struct Node { */ void reset() { expired_ = false; reply_time = time_point::min(); } + /** + * Generates a new request id, skipping the invalid id. + * + * @return the new id. + */ + uint16_t getNewTid() { + ++transaction_id; + return transaction_id == net::TransId::INVALID ? ++transaction_id : transaction_id; + } + std::string toString() const; OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const Node& h); @@ -138,6 +148,8 @@ private: unsigned auth_errors {0}; bool expired_ {false}; + uint16_t transaction_id {1}; + using TransactionDist = std::uniform_int_distribution; }; } diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 080fa737..40c6b5fd 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -196,7 +196,6 @@ NetworkEngine::NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& l if (!set_nonblocking(dht_socket6, 1)) throw DhtException("Can't set socket to non-blocking mode"); } - transaction_id = std::uniform_int_distribution{1}(rd_device); } NetworkEngine::~NetworkEngine() { @@ -629,7 +628,7 @@ NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr Sp NetworkEngine::sendPing(Sp node, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::PING, getNewTid()}; + auto tid = TransId {TransPrefix::PING, node->getNewTid()}; msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -689,7 +688,7 @@ NetworkEngine::sendPong(const SockAddr& addr, TransId tid) { Sp NetworkEngine::sendFindNode(Sp n, const InfoHash& target, want_t want, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::FIND_NODE, getNewTid()}; + auto tid = TransId {TransPrefix::FIND_NODE, n->getNewTid()}; msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -735,7 +734,7 @@ NetworkEngine::sendFindNode(Sp n, const InfoHash& target, want_t want, Sp NetworkEngine::sendGetValues(Sp n, const InfoHash& info_hash, const Query& query, want_t want, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::GET_VALUES, getNewTid()}; + auto tid = TransId {TransPrefix::GET_VALUES, n->getNewTid()}; msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -1011,13 +1010,13 @@ NetworkEngine::sendListen(Sp n, SocketCb&& socket_cb) { Sp socket; - auto tid = TransId { TransPrefix::LISTEN, previous ? previous->tid.getTid() : getNewTid() }; + auto tid = TransId { TransPrefix::LISTEN, previous ? previous->tid.getTid() : n->getNewTid() }; if (previous and previous->node == n) { socket = previous->socket; } else { if (previous) DHT_LOG.e(hash, "[node %s] trying refresh listen contract with wrong node", previous->node->toString().c_str()); - socket = n->openSocket(TransId {TransPrefix::GET_VALUES, getNewTid()}, std::move(socket_cb)); + socket = n->openSocket(TransPrefix::GET_VALUES, std::move(socket_cb)); } if (not socket) { @@ -1096,7 +1095,7 @@ NetworkEngine::sendAnnounceValue(Sp n, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::ANNOUNCE_VALUES, getNewTid()}; + auto tid = TransId {TransPrefix::ANNOUNCE_VALUES, n->getNewTid()}; msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -1154,7 +1153,7 @@ NetworkEngine::sendRefreshValue(Sp n, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::REFRESH, getNewTid()}; + auto tid = TransId {TransPrefix::REFRESH, n->getNewTid()}; msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); diff --git a/src/node.cpp b/src/node.cpp index a6bcc77e..6bb25d66 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -20,6 +20,7 @@ #include "node.h" #include "request.h" +#include "rng.h" #include @@ -29,6 +30,15 @@ constexpr std::chrono::minutes Node::NODE_EXPIRE_TIME; constexpr std::chrono::minutes Node::NODE_GOOD_TIME; constexpr std::chrono::seconds Node::MAX_RESPONSE_TIME; + +Node::Node(const InfoHash& id, const SockAddr& addr) +: id(id), + addr(addr) +{ + crypto::random_device rd; + transaction_id = TransactionDist{1}(rd); +} + /* This is our definition of a known-good node. */ bool Node::isGood(time_point now) const diff --git a/src/request.h b/src/request.h index 4604059f..d75554f1 100644 --- a/src/request.h +++ b/src/request.h @@ -76,7 +76,7 @@ struct Request { State getState() const { return state_; } Request(State state = State::PENDING) : state_(state) {} - Request(TransId tid, + Request(const TransId& tid, Sp node, Blob&& msg, std::function on_done, From 024dd42e5a002aea47776782c683f137a893af0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 11 Oct 2017 17:12:29 +0200 Subject: [PATCH 027/156] dht: remove processMessage --- include/opendht/dht.h | 2 -- src/dht.cpp | 29 +++++++---------------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/include/opendht/dht.h b/include/opendht/dht.h index 50215b47..3f70e371 100644 --- a/include/opendht/dht.h +++ b/include/opendht/dht.h @@ -553,8 +553,6 @@ private: bool neighbourhoodMaintenance(RoutingTable&); - void processMessage(const uint8_t *buf, size_t buflen, const SockAddr&); - void onError(Sp node, net::DhtProtocolException e); /* when our address is reported by a distant peer. */ void onReportedAddr(const InfoHash& id, const SockAddr&); diff --git a/src/dht.cpp b/src/dht.cpp index 35a8ff19..cc3e79f5 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -1863,32 +1863,17 @@ Dht::maintainStorage(decltype(store)::value_type& storage, bool force, DoneCallb return announce_per_af; } -void -Dht::processMessage(const uint8_t *buf, size_t buflen, const SockAddr& from) -{ - if (buflen == 0) - return; - - try { - network_engine.processMessage(buf, buflen, from); - } catch (const std::exception& e) { - DHT_LOG.e("Can't parse message from %s: %s", from.toString().c_str(), e.what()); - //auto code = e.getCode(); - //if (code == net::DhtProtocolException::INVALID_TID_SIZE or code == net::DhtProtocolException::WRONG_NODE_INFO_BUF_LEN) { - /* This is really annoying, as it means that we will - time-out all our searches that go through this node. - Kill it. */ - //const auto& id = e.getNodeId(); - //blacklistNode(&id, from, fromlen); - ///} - } -} - time_point Dht::periodic(const uint8_t *buf, size_t buflen, const SockAddr& from) { scheduler.syncTime(); - processMessage(buf, buflen, from); + if (buflen) { + try { + network_engine.processMessage(buf, buflen, from); + } catch (const std::exception& e) { + DHT_LOG.e("Can't parse message from %s: %s", from.toString().c_str(), e.what()); + } + } return scheduler.run(); } From 4ce01689166f201c552e45b4aa3da351f9ef5ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 11 Oct 2017 23:33:32 +0200 Subject: [PATCH 028/156] network engine: move ParsedMessage to separate file --- CMakeLists.txt | 1 + src/Makefile.am | 2 +- src/network_engine.cpp | 275 +------------------------------------ src/parsed_message.h | 303 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 306 insertions(+), 275 deletions(-) create mode 100644 src/parsed_message.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a7241de..da8731da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,7 @@ list (APPEND opendht_SOURCES src/storage.h src/listener.h src/search.h + src/parsed_message.h src/request.h src/callbacks.cpp src/routing_table.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 60536d99..634f2d88 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,6 +9,7 @@ libopendht_la_SOURCES = \ listener.h \ request.h \ search.h \ + parsed_message.h \ node_cache.cpp \ callbacks.cpp \ routing_table.cpp \ @@ -82,4 +83,3 @@ noinst_HEADERS = \ @top_builddir@/argon2/src/encoding.h \ @top_builddir@/argon2/src/thread.h endif - diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 40c6b5fd..2b02c427 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -22,6 +22,7 @@ #include "request.h" #include "default_types.h" #include "log_enable.h" +#include "parsed_message.h" #include @@ -95,63 +96,6 @@ set_nonblocking(int fd, int nonblocking) #endif -enum class MessageType { - Error = 0, - Reply, - Ping, - FindNode, - GetValues, - AnnounceValue, - Refresh, - Listen, - ValueData -}; - -struct ParsedMessage { - MessageType type; - /* Node ID of the sender */ - InfoHash id; - /* Network id */ - NetId network {0}; - /* hash for which values are requested */ - InfoHash info_hash; - /* target id around which to find nodes */ - InfoHash target; - /* transaction id */ - TransId tid; - /* tid for packets going through request socket */ - TransId socket_id; - /* security token */ - Blob token; - /* the value id (announce confirmation) */ - Value::Id value_id; - /* time when value was first created */ - time_point created { time_point::max() }; - /* IPv4 nodes in response to a 'find' request */ - Blob nodes4_raw, nodes6_raw; - std::vector> nodes4, nodes6; - /* values to store or retreive request */ - std::vector> values; - /* index for fields values */ - std::vector> fields; - /** When part of the message header: {index -> (total size, {})} - * When part of partial value data: {index -> (offset, part_data)} */ - std::map> value_parts; - /* query describing a filter to apply on values. */ - Query query; - /* states if ipv4 or ipv6 request */ - want_t want; - /* error code in case of error */ - uint16_t error_code; - /* reported address by the distant node */ - std::string ua; - SockAddr addr; - void msgpack_unpack(msgpack::object o); - - bool append(const ParsedMessage& block); - bool complete(); -}; - struct NetworkEngine::PartialMessage { SockAddr from; time_point start; @@ -1250,185 +1194,6 @@ NetworkEngine::sendError(const SockAddr& addr, send(buffer.data(), buffer.size(), 0, addr); } -void -ParsedMessage::msgpack_unpack(msgpack::object msg) -{ - auto y = findMapValue(msg, "y"); - auto r = findMapValue(msg, "r"); - auto e = findMapValue(msg, "e"); - auto v = findMapValue(msg, "p"); - - if (auto t = findMapValue(msg, "t")) - tid = {t->as>()}; - - if (auto rv = findMapValue(msg, "v")) - ua = rv->as(); - - if (auto netid = findMapValue(msg, "n")) - network = netid->as(); - - std::string q; - if (auto rq = findMapValue(msg, "q")) { - if (rq->type != msgpack::type::STR) - throw msgpack::type_error(); - q = rq->as(); - } - - if (e) - type = MessageType::Error; - else if (r) - type = MessageType::Reply; - else if (v) - type = MessageType::ValueData; - else if (y and y->as() != "q") - throw msgpack::type_error(); - else if (q == "ping") - type = MessageType::Ping; - else if (q == "find") - type = MessageType::FindNode; - else if (q == "get") - type = MessageType::GetValues; - else if (q == "listen") - type = MessageType::Listen; - else if (q == "put") - type = MessageType::AnnounceValue; - else if (q == "refresh") - type = MessageType::Refresh; - else - throw msgpack::type_error(); - - if (type == MessageType::ValueData) { - if (v->type != msgpack::type::MAP) - throw msgpack::type_error(); - for (size_t i = 0; i < v->via.map.size; ++i) { - auto& vdat = v->via.map.ptr[i]; - auto o = findMapValue(vdat.val, "o"); - auto d = findMapValue(vdat.val, "d"); - if (not o or not d) - continue; - value_parts.emplace(vdat.key.as(), std::pair(o->as(), unpackBlob(*d))); - } - return; - } - - auto a = findMapValue(msg, "a"); - if (!a && !r && !e) - throw msgpack::type_error(); - auto& req = a ? *a : (r ? *r : *e); - - if (e) { - if (e->type != msgpack::type::ARRAY) - throw msgpack::type_error(); - error_code = e->via.array.ptr[0].as(); - } - - if (auto t = findMapValue(req, "sid")) - socket_id = {t->as>()}; - - if (auto rid = findMapValue(req, "id")) - id = {*rid}; - - if (auto rh = findMapValue(req, "h")) - info_hash = {*rh}; - - if (auto rtarget = findMapValue(req, "target")) - target = {*rtarget}; - - if (auto rquery = findMapValue(req, "q")) - query.msgpack_unpack(*rquery); - - if (auto otoken = findMapValue(req, "token")) - token = unpackBlob(*otoken); - - if (auto vid = findMapValue(req, "vid")) - value_id = vid->as(); - - if (auto rnodes4 = findMapValue(req, "n4")) - nodes4_raw = unpackBlob(*rnodes4); - - if (auto rnodes6 = findMapValue(req, "n6")) - nodes6_raw = unpackBlob(*rnodes6); - - if (auto sa = findMapValue(req, "sa")) { - if (sa->type != msgpack::type::BIN) - throw msgpack::type_error(); - auto l = sa->via.bin.size; - if (l == sizeof(in_addr)) { - addr.setFamily(AF_INET); - auto& a = addr.getIPv4(); - std::fill_n((uint8_t*)&a, sizeof(a), 0); - a.sin_port = 0; - std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin_addr); - } else if (l == sizeof(in6_addr)) { - addr.setFamily(AF_INET6); - auto& a = addr.getIPv6(); - std::fill_n((uint8_t*)&a, sizeof(a), 0); - a.sin6_port = 0; - std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin6_addr); - } - } else - addr = {}; - - if (auto rcreated = findMapValue(req, "c")) - created = from_time_t(rcreated->as()); - - if (auto rvalues = findMapValue(req, "values")) { - if (rvalues->type != msgpack::type::ARRAY) - throw msgpack::type_error(); - for (size_t i = 0; i < rvalues->via.array.size; i++) { - auto& packed_v = rvalues->via.array.ptr[i]; - if (packed_v.type == msgpack::type::POSITIVE_INTEGER) { - // Skip oversize values with a small margin for header overhead - if (packed_v.via.u64 > MAX_VALUE_SIZE + 32) - continue; - value_parts.emplace(i, std::make_pair(packed_v.via.u64, Blob{})); - } else { - try { - values.emplace_back(std::make_shared(rvalues->via.array.ptr[i])); - } catch (const std::exception& e) { - //DHT_LOG_WARN("Error reading value: %s", e.what()); - } - } - } - } else if (auto raw_fields = findMapValue(req, "fields")) { - if (auto rfields = findMapValue(*raw_fields, "f")) { - auto vfields = rfields->as>(); - if (auto rvalues = findMapValue(*raw_fields, "v")) { - if (rvalues->type != msgpack::type::ARRAY) - throw msgpack::type_error(); - size_t val_num = rvalues->via.array.size / vfields.size(); - for (size_t i = 0; i < val_num; ++i) { - try { - auto v = std::make_shared(); - v->msgpack_unpack_fields(vfields, *rvalues, i*vfields.size()); - fields.emplace_back(std::move(v)); - } catch (const std::exception& e) { } - } - } - } else { - throw msgpack::type_error(); - } - } - - if (auto w = findMapValue(req, "w")) { - if (w->type != msgpack::type::ARRAY) - throw msgpack::type_error(); - want = 0; - for (unsigned i=0; ivia.array.size; i++) { - auto& val = w->via.array.ptr[i]; - try { - auto w = val.as(); - if (w == AF_INET) - want |= WANT4; - else if(w == AF_INET6) - want |= WANT6; - } catch (const std::exception& e) {}; - } - } else { - want = -1; - } -} - void NetworkEngine::maintainRxBuffer(const TransId& tid) { @@ -1443,44 +1208,6 @@ NetworkEngine::maintainRxBuffer(const TransId& tid) } } -bool -ParsedMessage::append(const ParsedMessage& block) -{ - bool ret(false); - for (const auto& ve : block.value_parts) { - auto part_val = value_parts.find(ve.first); - if (part_val == value_parts.end() - || part_val->second.second.size() >= part_val->second.first) - continue; - // TODO: handle out-of-order packets - if (ve.second.first != part_val->second.second.size()) { - //std::cout << "skipping out-of-order packet" << std::endl; - continue; - } - ret = true; - part_val->second.second.insert(part_val->second.second.end(), - ve.second.second.begin(), - ve.second.second.end()); - } - return ret; -} - -bool -ParsedMessage::complete() -{ - for (auto& e : value_parts) { - //std::cout << "part " << e.first << ": " << e.second.second.size() << "/" << e.second.first << std::endl; - if (e.second.first > e.second.second.size()) - return false; - } - for (auto& e : value_parts) { - msgpack::unpacked msg; - msgpack::unpack(msg, (const char*)e.second.second.data(), e.second.second.size()); - values.emplace_back(std::make_shared(msg.get())); - } - return true; -} - } /* namespace net */ } /* namespace dht */ diff --git a/src/parsed_message.h b/src/parsed_message.h new file mode 100644 index 00000000..83c74ab2 --- /dev/null +++ b/src/parsed_message.h @@ -0,0 +1,303 @@ +/* + * Copyright (C) 2017 Savoir-faire Linux Inc. + * Author(s) : Adrien Béraud + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + #pragma once + +#include "infohash.h" +#include "sockaddr.h" + +#include + +namespace dht { +namespace net { + +enum class MessageType { + Error = 0, + Reply, + Ping, + FindNode, + GetValues, + AnnounceValue, + Refresh, + Listen, + ValueData +}; + +struct ParsedMessage { + MessageType type; + /* Node ID of the sender */ + InfoHash id; + /* Network id */ + NetId network {0}; + /* hash for which values are requested */ + InfoHash info_hash; + /* target id around which to find nodes */ + InfoHash target; + /* transaction id */ + TransId tid; + /* tid for packets going through request socket */ + TransId socket_id; + /* security token */ + Blob token; + /* the value id (announce confirmation) */ + Value::Id value_id; + /* time when value was first created */ + time_point created { time_point::max() }; + /* IPv4 nodes in response to a 'find' request */ + Blob nodes4_raw, nodes6_raw; + std::vector> nodes4, nodes6; + /* values to store or retreive request */ + std::vector> values; + /* index for fields values */ + std::vector> fields; + /** When part of the message header: {index -> (total size, {})} + * When part of partial value data: {index -> (offset, part_data)} */ + std::map> value_parts; + /* query describing a filter to apply on values. */ + Query query; + /* states if ipv4 or ipv6 request */ + want_t want; + /* error code in case of error */ + uint16_t error_code; + /* reported address by the distant node */ + std::string ua; + SockAddr addr; + void msgpack_unpack(msgpack::object o); + + bool append(const ParsedMessage& block); + bool complete(); +}; + +bool +ParsedMessage::append(const ParsedMessage& block) +{ + bool ret(false); + for (const auto& ve : block.value_parts) { + auto part_val = value_parts.find(ve.first); + if (part_val == value_parts.end() + || part_val->second.second.size() >= part_val->second.first) + continue; + // TODO: handle out-of-order packets + if (ve.second.first != part_val->second.second.size()) { + //std::cout << "skipping out-of-order packet" << std::endl; + continue; + } + ret = true; + part_val->second.second.insert(part_val->second.second.end(), + ve.second.second.begin(), + ve.second.second.end()); + } + return ret; +} + +bool +ParsedMessage::complete() +{ + for (auto& e : value_parts) { + //std::cout << "part " << e.first << ": " << e.second.second.size() << "/" << e.second.first << std::endl; + if (e.second.first > e.second.second.size()) + return false; + } + for (auto& e : value_parts) { + msgpack::unpacked msg; + msgpack::unpack(msg, (const char*)e.second.second.data(), e.second.second.size()); + values.emplace_back(std::make_shared(msg.get())); + } + return true; +} + +void +ParsedMessage::msgpack_unpack(msgpack::object msg) +{ + auto y = findMapValue(msg, "y"); + auto r = findMapValue(msg, "r"); + auto e = findMapValue(msg, "e"); + auto v = findMapValue(msg, "p"); + + if (auto t = findMapValue(msg, "t")) + tid = {t->as>()}; + + if (auto rv = findMapValue(msg, "v")) + ua = rv->as(); + + if (auto netid = findMapValue(msg, "n")) + network = netid->as(); + + std::string q; + if (auto rq = findMapValue(msg, "q")) { + if (rq->type != msgpack::type::STR) + throw msgpack::type_error(); + q = rq->as(); + } + + if (e) + type = MessageType::Error; + else if (r) + type = MessageType::Reply; + else if (v) + type = MessageType::ValueData; + else if (y and y->as() != "q") + throw msgpack::type_error(); + else if (q == "ping") + type = MessageType::Ping; + else if (q == "find") + type = MessageType::FindNode; + else if (q == "get") + type = MessageType::GetValues; + else if (q == "listen") + type = MessageType::Listen; + else if (q == "put") + type = MessageType::AnnounceValue; + else if (q == "refresh") + type = MessageType::Refresh; + else + throw msgpack::type_error(); + + if (type == MessageType::ValueData) { + if (v->type != msgpack::type::MAP) + throw msgpack::type_error(); + for (size_t i = 0; i < v->via.map.size; ++i) { + auto& vdat = v->via.map.ptr[i]; + auto o = findMapValue(vdat.val, "o"); + auto d = findMapValue(vdat.val, "d"); + if (not o or not d) + continue; + value_parts.emplace(vdat.key.as(), std::pair(o->as(), unpackBlob(*d))); + } + return; + } + + auto a = findMapValue(msg, "a"); + if (!a && !r && !e) + throw msgpack::type_error(); + auto& req = a ? *a : (r ? *r : *e); + + if (e) { + if (e->type != msgpack::type::ARRAY) + throw msgpack::type_error(); + error_code = e->via.array.ptr[0].as(); + } + + if (auto t = findMapValue(req, "sid")) + socket_id = {t->as>()}; + + if (auto rid = findMapValue(req, "id")) + id = {*rid}; + + if (auto rh = findMapValue(req, "h")) + info_hash = {*rh}; + + if (auto rtarget = findMapValue(req, "target")) + target = {*rtarget}; + + if (auto rquery = findMapValue(req, "q")) + query.msgpack_unpack(*rquery); + + if (auto otoken = findMapValue(req, "token")) + token = unpackBlob(*otoken); + + if (auto vid = findMapValue(req, "vid")) + value_id = vid->as(); + + if (auto rnodes4 = findMapValue(req, "n4")) + nodes4_raw = unpackBlob(*rnodes4); + + if (auto rnodes6 = findMapValue(req, "n6")) + nodes6_raw = unpackBlob(*rnodes6); + + if (auto sa = findMapValue(req, "sa")) { + if (sa->type != msgpack::type::BIN) + throw msgpack::type_error(); + auto l = sa->via.bin.size; + if (l == sizeof(in_addr)) { + addr.setFamily(AF_INET); + auto& a = addr.getIPv4(); + std::fill_n((uint8_t*)&a, sizeof(a), 0); + a.sin_port = 0; + std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin_addr); + } else if (l == sizeof(in6_addr)) { + addr.setFamily(AF_INET6); + auto& a = addr.getIPv6(); + std::fill_n((uint8_t*)&a, sizeof(a), 0); + a.sin6_port = 0; + std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin6_addr); + } + } else + addr = {}; + + if (auto rcreated = findMapValue(req, "c")) + created = from_time_t(rcreated->as()); + + if (auto rvalues = findMapValue(req, "values")) { + if (rvalues->type != msgpack::type::ARRAY) + throw msgpack::type_error(); + for (size_t i = 0; i < rvalues->via.array.size; i++) { + auto& packed_v = rvalues->via.array.ptr[i]; + if (packed_v.type == msgpack::type::POSITIVE_INTEGER) { + // Skip oversize values with a small margin for header overhead + if (packed_v.via.u64 > MAX_VALUE_SIZE + 32) + continue; + value_parts.emplace(i, std::make_pair(packed_v.via.u64, Blob{})); + } else { + try { + values.emplace_back(std::make_shared(rvalues->via.array.ptr[i])); + } catch (const std::exception& e) { + //DHT_LOG_WARN("Error reading value: %s", e.what()); + } + } + } + } else if (auto raw_fields = findMapValue(req, "fields")) { + if (auto rfields = findMapValue(*raw_fields, "f")) { + auto vfields = rfields->as>(); + if (auto rvalues = findMapValue(*raw_fields, "v")) { + if (rvalues->type != msgpack::type::ARRAY) + throw msgpack::type_error(); + size_t val_num = rvalues->via.array.size / vfields.size(); + for (size_t i = 0; i < val_num; ++i) { + try { + auto v = std::make_shared(); + v->msgpack_unpack_fields(vfields, *rvalues, i*vfields.size()); + fields.emplace_back(std::move(v)); + } catch (const std::exception& e) { } + } + } + } else { + throw msgpack::type_error(); + } + } + + if (auto w = findMapValue(req, "w")) { + if (w->type != msgpack::type::ARRAY) + throw msgpack::type_error(); + want = 0; + for (unsigned i=0; ivia.array.size; i++) { + auto& val = w->via.array.ptr[i]; + try { + auto w = val.as(); + if (w == AF_INET) + want |= WANT4; + else if(w == AF_INET6) + want |= WANT6; + } catch (const std::exception& e) {}; + } + } else { + want = -1; + } +} + +} /* namespace net */ +} /* namespace dht */ From e20ae56cabc02b63e4af9ea6d825b6d0b6188179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 12 Oct 2017 01:02:24 +0200 Subject: [PATCH 029/156] dht: support client mode In client mode, a node won't store data and won't participate in routing. --- include/opendht/node.h | 4 +++- include/opendht/node_cache.h | 5 +++-- src/network_engine.cpp | 12 +++++++----- src/node.cpp | 5 ++--- src/node_cache.cpp | 16 ++++++++-------- src/parsed_message.h | 5 +++++ 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/include/opendht/node.h b/include/opendht/node.h index fa00d4dd..0332a3a5 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -40,11 +40,12 @@ using SocketCb = std::function&, RequestAnswer&&)>; struct Node { InfoHash id; SockAddr addr; + bool is_client {false}; time_point time {time_point::min()}; /* last time eared about */ time_point reply_time {time_point::min()}; /* time of last correct reply received */ - Node(const InfoHash& id, const SockAddr& addr); + Node(const InfoHash& id, const SockAddr& addr, bool client=false); Node(const InfoHash& id, const sockaddr* sa, socklen_t salen) : Node(id, SockAddr(sa, salen)) {} @@ -57,6 +58,7 @@ struct Node { std::string getAddrStr() const { return addr.toString(); } + bool isClient() const { return is_client; } /** * Makes notice about an additionnal authentication error with this node. Up diff --git a/include/opendht/node_cache.h b/include/opendht/node_cache.h index d18b7664..e5f90868 100644 --- a/include/opendht/node_cache.h +++ b/include/opendht/node_cache.h @@ -27,7 +27,7 @@ namespace dht { struct NodeCache { std::shared_ptr getNode(const InfoHash& id, sa_family_t family); - std::shared_ptr getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed); + std::shared_ptr getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client=false); std::vector> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count); /** @@ -41,10 +41,11 @@ private: class NodeMap : public std::map> { public: std::shared_ptr getNode(const InfoHash& id); - std::shared_ptr getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed); + std::shared_ptr getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client); void clearBadNodes(); }; + NodeMap& cache(sa_family_t af) { return af == AF_INET ? cache_4 : cache_6; } NodeMap cache_4; NodeMap cache_6; }; diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 2b02c427..4c25e726 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -399,9 +399,9 @@ void NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& from) { const auto& now = scheduler.time(); + auto node = cache.getNode(msg->id, from, now, true, msg->is_client); if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { - auto node = cache.getNode(msg->id, from, now, true); auto rsocket = node->getSocket(msg->tid); auto req = node->getRequest(msg->tid); @@ -414,14 +414,16 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro requests.erase(req_it); } else { node->received(now, req); - onNewNode(node, 1); + if (not node->is_client) + onNewNode(node, 1); throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find transaction", msg->id}; } } node->received(now, req); - onNewNode(node, 2); + if (not node->is_client) + onNewNode(node, 2); onReportedAddr(msg->id, msg->addr); if (req and (req->cancelled() or req->expired() or req->completed())) { @@ -463,9 +465,9 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro break; } } else { - auto node = cache.getNode(msg->id, from, now, true); node->received(now, {}); - onNewNode(node, 1); + if (not node->is_client) + onNewNode(node, 1); try { switch (msg->type) { case MessageType::Ping: diff --git a/src/node.cpp b/src/node.cpp index 6bb25d66..26cdb8de 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -31,9 +31,8 @@ constexpr std::chrono::minutes Node::NODE_GOOD_TIME; constexpr std::chrono::seconds Node::MAX_RESPONSE_TIME; -Node::Node(const InfoHash& id, const SockAddr& addr) -: id(id), - addr(addr) +Node::Node(const InfoHash& id, const SockAddr& addr, bool client) +: id(id), addr(addr), is_client(client) { crypto::random_device rd; transaction_id = TransactionDist{1}(rd); diff --git a/src/node_cache.cpp b/src/node_cache.cpp index 4bc51215..f3a92334 100644 --- a/src/node_cache.cpp +++ b/src/node_cache.cpp @@ -22,19 +22,19 @@ namespace dht { std::shared_ptr NodeCache::getNode(const InfoHash& id, sa_family_t family) { - return (family == AF_INET ? cache_4 : cache_6).getNode(id); + return cache(family).getNode(id); } std::shared_ptr -NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm) { +NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) { if (id == zeroes) return std::make_shared(id, addr); - return (addr.getFamily() == AF_INET ? cache_4 : cache_6).getNode(id, addr, now, confirm); + return cache(addr.getFamily()).getNode(id, addr, now, confirm, client); } std::vector> NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) { - const auto& c = (sa_f == AF_INET ? cache_4 : cache_6); + const auto& c = cache(sa_f); auto it_p = c.lower_bound(id), it_n = it_p; @@ -57,7 +57,7 @@ NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) { it_p = c.end(); if (auto n = it->second.lock()) - if ( not n->isExpired() ) + if ( not n->isExpired() and not n->isClient() ) nodes.emplace_back(std::move(n)); } @@ -71,7 +71,7 @@ NodeCache::clearBadNodes(sa_family_t family) clearBadNodes(AF_INET); clearBadNodes(AF_INET6); } else { - (family == AF_INET ? cache_4 : cache_6).clearBadNodes(); + cache(family).clearBadNodes(); } } @@ -88,12 +88,12 @@ NodeCache::NodeMap::getNode(const InfoHash& id) } std::shared_ptr -NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm) +NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) { auto it = emplace(id, std::weak_ptr{}); auto node = it.first->second.lock(); if (not node) { - node = std::make_shared(id, addr); + node = std::make_shared(id, addr, client); it.first->second = node; } else if (confirm || node->time < now - Node::NODE_EXPIRE_TIME) { node->update(addr); diff --git a/src/parsed_message.h b/src/parsed_message.h index 83c74ab2..f2f691d6 100644 --- a/src/parsed_message.h +++ b/src/parsed_message.h @@ -43,6 +43,8 @@ struct ParsedMessage { InfoHash id; /* Network id */ NetId network {0}; + /** Is a client node */ + bool is_client {false}; /* hash for which values are requested */ InfoHash info_hash; /* target id around which to find nodes */ @@ -137,6 +139,9 @@ ParsedMessage::msgpack_unpack(msgpack::object msg) if (auto netid = findMapValue(msg, "n")) network = netid->as(); + if (auto is_client_v = findMapValue(msg, "s")) + is_client = is_client_v->as(); + std::string q; if (auto rq = findMapValue(msg, "q")) { if (rq->type != msgpack::type::STR) From 84a45c4f9185ea9baa4c072fff28db471555ce76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 12 Oct 2017 15:43:42 +0200 Subject: [PATCH 030/156] dht: factorize shared pointer dereference --- src/dht.cpp | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index cc3e79f5..3cf58fb3 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -2234,29 +2234,30 @@ Dht::onListenDone(const Sp& node, } net::RequestAnswer -Dht::onAnnounce(Sp node, +Dht::onAnnounce(Sp n, const InfoHash& hash, const Blob& token, const std::vector>& values, const time_point& creation_date) { + auto& node = *n; if (hash == zeroes) { - DHT_LOG.w(node->id, "put with no info_hash"); + DHT_LOG.w(node.id, "put with no info_hash"); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, net::DhtProtocolException::PUT_NO_INFOHASH }; } - if (!tokenMatch(token, node->addr)) { - DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'put'", node->toString().c_str(), hash.toString().c_str()); + if (!tokenMatch(token, node.addr)) { + DHT_LOG.w(hash, node.id, "[node %s] incorrect token %s for 'put'", node.toString().c_str(), hash.toString().c_str()); throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::PUT_WRONG_TOKEN}; } { // We store a value only if we think we're part of the // SEARCH_NODES nodes around the target id. - auto closest_nodes = buckets(node->getFamily()).findClosestNodes(hash, scheduler.time(), SEARCH_NODES); + auto closest_nodes = buckets(node.getFamily()).findClosestNodes(hash, scheduler.time(), SEARCH_NODES); if (closest_nodes.size() >= TARGET_NODES and hash.xorCmp(closest_nodes.back()->id, myid) < 0) { - DHT_LOG.w(hash, node->id, "[node %s] announce too far from the target. Dropping value.", node->toString().c_str()); + DHT_LOG.w(hash, node.id, "[node %s] announce too far from the target. Dropping value.", node.toString().c_str()); return {}; } } @@ -2264,7 +2265,7 @@ Dht::onAnnounce(Sp node, auto created = std::min(creation_date, scheduler.time()); for (const auto& v : values) { if (v->id == Value::INVALID_ID) { - DHT_LOG.w(hash, node->id, "[value %s] incorrect value id", hash.toString().c_str()); + DHT_LOG.w(hash, node.id, "[value %s] incorrect value id", hash.toString().c_str()); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, net::DhtProtocolException::PUT_INVALID_ID @@ -2274,26 +2275,26 @@ Dht::onAnnounce(Sp node, Sp vc = v; if (lv) { if (*lv == *vc) { - DHT_LOG.w(hash, node->id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str()); + DHT_LOG.d(hash, node.id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str()); } else { const auto& type = getType(lv->type); - if (type.editPolicy(hash, lv, vc, node->id, node->addr.get(), node->addr.getLength())) { - DHT_LOG.d(hash, node->id, "[store %s] editing %s", + if (type.editPolicy(hash, lv, vc, node.id, node.addr.get(), node.addr.getLength())) { + DHT_LOG.d(hash, node.id, "[store %s] editing %s", hash.toString().c_str(), vc->toString().c_str()); - storageStore(hash, vc, created, &node->addr); + storageStore(hash, vc, created, &node.addr); } else { - DHT_LOG.d(hash, node->id, "[store %s] rejecting edition of %s because of storage policy", + DHT_LOG.d(hash, node.id, "[store %s] rejecting edition of %s because of storage policy", hash.toString().c_str(), vc->toString().c_str()); } } } else { // Allow the value to be edited by the storage policy const auto& type = getType(vc->type); - if (type.storePolicy(hash, vc, node->id, node->addr.get(), node->addr.getLength())) { - DHT_LOG.d(hash, node->id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str()); - storageStore(hash, vc, created, &node->addr); + if (type.storePolicy(hash, vc, node.id, node.addr.get(), node.addr.getLength())) { + DHT_LOG.d(hash, node.id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str()); + storageStore(hash, vc, created, &node.addr); } else { - DHT_LOG.d(hash, node->id, "[store %s] rejecting storage of %s", + DHT_LOG.d(hash, node.id, "[store %s] rejecting storage of %s", hash.toString().c_str(), vc->toString().c_str()); } } From 6ce1531e1fea7d58ac0b6abe9bd384b7ea7cb420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 12 Oct 2017 15:44:25 +0200 Subject: [PATCH 031/156] node: make id const --- include/opendht/node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/opendht/node.h b/include/opendht/node.h index 0332a3a5..9bf1937f 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -38,7 +38,7 @@ using SocketCb = std::function&, RequestAnswer&&)>; } /* namespace net */ struct Node { - InfoHash id; + const InfoHash id; SockAddr addr; bool is_client {false}; From e9f007025f6a145cad48f6504b9514afdf635606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 12 Oct 2017 17:15:52 +0200 Subject: [PATCH 032/156] node: make most fields private --- include/opendht/dht.h | 2 +- include/opendht/node.h | 31 ++++++++++++++++++++----------- src/dht.cpp | 38 ++++++++++++++++++++------------------ src/network_engine.cpp | 20 ++++++++++---------- src/node_cache.cpp | 2 +- src/search.h | 6 +++--- 6 files changed, 55 insertions(+), 44 deletions(-) diff --git a/include/opendht/dht.h b/include/opendht/dht.h index 3f70e371..9c8b38d6 100644 --- a/include/opendht/dht.h +++ b/include/opendht/dht.h @@ -421,7 +421,7 @@ private: // Storage void storageAddListener(const InfoHash& id, const Sp& node, size_t tid, Query&& = {}); - bool storageStore(const InfoHash& id, const Sp& value, time_point created, const SockAddr* sa = nullptr); + bool storageStore(const InfoHash& id, const Sp& value, time_point created, const SockAddr& sa = {}); void expireStore(); void expireStorage(InfoHash h); void expireStore(decltype(store)::iterator); diff --git a/include/opendht/node.h b/include/opendht/node.h index 9bf1937f..22e7e562 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -39,11 +39,6 @@ using SocketCb = std::function&, RequestAnswer&&)>; struct Node { const InfoHash id; - SockAddr addr; - bool is_client {false}; - - time_point time {time_point::min()}; /* last time eared about */ - time_point reply_time {time_point::min()}; /* time of last correct reply received */ Node(const InfoHash& id, const SockAddr& addr, bool client=false); Node(const InfoHash& id, const sockaddr* sa, socklen_t salen) @@ -52,13 +47,16 @@ struct Node { InfoHash getId() const { return id; } - std::pair getAddr() const { - return {addr.get(), addr.getLength()}; - } + const SockAddr& getAddr() const { return addr; } std::string getAddrStr() const { return addr.toString(); } bool isClient() const { return is_client; } + bool isIncoming() { return time > reply_time; } + + const time_point& getTime() const { return time; } + const time_point& getReplyTime() const { return reply_time; } + void setTime(const time_point& t) { time = t; } /** * Makes notice about an additionnal authentication error with this node. Up @@ -76,6 +74,13 @@ struct Node { bool isPendingMessage() const; size_t getPendingMessageCount() const; + bool isOld(const time_point& now) const { + return time + NODE_EXPIRE_TIME < now; + } + bool isRemovable(const time_point& now) const { + return isExpired() and isOld(now); + } + NodeExport exportNode() const { NodeExport ne; ne.id = id; @@ -145,13 +150,17 @@ private: /* Number of times we accept authentication errors from this node. */ static const constexpr unsigned MAX_AUTH_ERRORS {3}; - std::map> requests_ {}; - std::map> sockets_ {}; - + SockAddr addr; + bool is_client {false}; + time_point time {time_point::min()}; /* last time eared about */ + time_point reply_time {time_point::min()}; /* time of last correct reply received */ unsigned auth_errors {0}; bool expired_ {false}; uint16_t transaction_id {1}; using TransactionDist = std::uniform_int_distribution; + + std::map> requests_ {}; + std::map> sockets_ {}; }; } diff --git a/src/dht.cpp b/src/dht.cpp index 3cf58fb3..b42db9ae 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -1166,7 +1166,7 @@ Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v) node_listeners.first->toString().c_str()); std::vector> vals {}; vals.push_back(v.data); - Blob ntoken = makeToken(node_listeners.first->addr, false); + Blob ntoken = makeToken(node_listeners.first->getAddr(), false); network_engine.tellListener(node_listeners.first, l.second.sid, id, 0, ntoken, {}, {}, std::move(vals), l.second.query); } @@ -1174,7 +1174,7 @@ Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v) } bool -Dht::storageStore(const InfoHash& id, const Sp& value, time_point created, const SockAddr* sa) +Dht::storageStore(const InfoHash& id, const Sp& value, time_point created, const SockAddr& sa) { const auto& now = scheduler.time(); created = std::min(created, now); @@ -1194,7 +1194,7 @@ Dht::storageStore(const InfoHash& id, const Sp& value, time_point created StorageBucket* store_bucket {nullptr}; if (sa) - store_bucket = &store_quota.emplace(*sa, StorageBucket{}).first->second; + store_bucket = &store_quota.emplace(sa, StorageBucket{}).first->second; auto store = st->second.store(id, value, created, expiration, store_bucket); if (auto vs = store.first) { @@ -1225,7 +1225,7 @@ Dht::storageAddListener(const InfoHash& id, const Sp& node, size_t socket_ if (l == node_listeners->second.end()) { auto vals = st->second.get(query.where.getFilter()); if (not vals.empty()) { - network_engine.tellListener(node, socket_id, id, WANT4 | WANT6, makeToken(node->addr, false), + network_engine.tellListener(node, socket_id, id, WANT4 | WANT6, makeToken(node->getAddr(), false), buckets4.findClosestNodes(id, now, TARGET_NODES), buckets6.findClosestNodes(id, now, TARGET_NODES), std::move(vals), query); } @@ -1383,7 +1383,7 @@ Dht::getNodesStats(sa_family_t af) const for (auto& n : b.nodes) { if (n->isGood(now)) { stats.good_nodes++; - if (n->time > n->reply_time) + if (n->isIncoming()) stats.incoming_nodes++; } else if (not n->isExpired()) stats.dubious_nodes++; @@ -1406,10 +1406,12 @@ Dht::dumpBucket(const Bucket& b, std::ostream& out) const out << std::endl; for (auto& n : b.nodes) { out << " Node " << n->toString(); - if (n->time != n->reply_time) - out << " age " << duration_cast(now - n->time).count() << ", reply: " << duration_cast(now - n->reply_time).count(); + const auto& t = n->getTime(); + const auto& r = n->getReplyTime(); + if (t != r) + out << " age " << duration_cast(now - t).count() << ", reply: " << duration_cast(now - r).count(); else - out << " age " << duration_cast(now - n->time).count(); + out << " age " << duration_cast(now - t).count(); if (n->isExpired()) out << " [expired]"; else if (n->isGood(now)) @@ -1504,7 +1506,7 @@ Dht::dumpSearch(const Search& sr, std::ostream& out) const out << "] "; } } - out << n.node->addr.toString() << std::endl; + out << n.node->getAddrStr() << std::endl; } } @@ -2094,7 +2096,7 @@ Dht::onFindNode(Sp node, const InfoHash& target, want_t want) { const auto& now = scheduler.time(); net::RequestAnswer answer; - answer.ntoken = makeToken(node->addr, false); + answer.ntoken = makeToken(node->getAddr(), false); if (want & WANT4) answer.nodes4 = buckets4.findClosestNodes(target, now, TARGET_NODES); if (want & WANT6) @@ -2115,7 +2117,7 @@ Dht::onGetValues(Sp node, const InfoHash& hash, want_t, const Query& query const auto& now = scheduler.time(); net::RequestAnswer answer {}; auto st = store.find(hash); - answer.ntoken = makeToken(node->addr, false); + answer.ntoken = makeToken(node->getAddr(), false); answer.nodes4 = buckets4.findClosestNodes(hash, now, TARGET_NODES); answer.nodes6 = buckets6.findClosestNodes(hash, now, TARGET_NODES); if (st != store.end() && not st->second.empty()) { @@ -2209,7 +2211,7 @@ Dht::onListen(Sp node, const InfoHash& hash, const Blob& token, size_t soc net::DhtProtocolException::LISTEN_NO_INFOHASH }; } - if (!tokenMatch(token, node->addr)) { + if (!tokenMatch(token, node->getAddr())) { DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'listen'", node->toString().c_str(), hash.toString().c_str()); throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::LISTEN_WRONG_TOKEN}; } @@ -2248,7 +2250,7 @@ Dht::onAnnounce(Sp n, net::DhtProtocolException::PUT_NO_INFOHASH }; } - if (!tokenMatch(token, node.addr)) { + if (!tokenMatch(token, node.getAddr())) { DHT_LOG.w(hash, node.id, "[node %s] incorrect token %s for 'put'", node.toString().c_str(), hash.toString().c_str()); throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::PUT_WRONG_TOKEN}; } @@ -2278,10 +2280,10 @@ Dht::onAnnounce(Sp n, DHT_LOG.d(hash, node.id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str()); } else { const auto& type = getType(lv->type); - if (type.editPolicy(hash, lv, vc, node.id, node.addr.get(), node.addr.getLength())) { + if (type.editPolicy(hash, lv, vc, node.id, node.getAddr().get(), node.getAddr().getLength())) { DHT_LOG.d(hash, node.id, "[store %s] editing %s", hash.toString().c_str(), vc->toString().c_str()); - storageStore(hash, vc, created, &node.addr); + storageStore(hash, vc, created, node.getAddr()); } else { DHT_LOG.d(hash, node.id, "[store %s] rejecting edition of %s because of storage policy", hash.toString().c_str(), vc->toString().c_str()); @@ -2290,9 +2292,9 @@ Dht::onAnnounce(Sp n, } else { // Allow the value to be edited by the storage policy const auto& type = getType(vc->type); - if (type.storePolicy(hash, vc, node.id, node.addr.get(), node.addr.getLength())) { + if (type.storePolicy(hash, vc, node.id, node.getAddr().get(), node.getAddr().getLength())) { DHT_LOG.d(hash, node.id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str()); - storageStore(hash, vc, created, &node.addr); + storageStore(hash, vc, created, node.getAddr()); } else { DHT_LOG.d(hash, node.id, "[store %s] rejecting storage of %s", hash.toString().c_str(), vc->toString().c_str()); @@ -2308,7 +2310,7 @@ Dht::onRefresh(Sp node, const InfoHash& hash, const Blob& token, const Val using namespace net; const auto& now = scheduler.time(); - if (not tokenMatch(token, node->addr)) { + if (not tokenMatch(token, node->getAddr())) { DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'put'", node->toString().c_str(), hash.toString().c_str()); throw DhtProtocolException {DhtProtocolException::UNAUTHORIZED, DhtProtocolException::PUT_WRONG_TOKEN}; } diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 4c25e726..8843ee3f 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -154,7 +154,7 @@ NetworkEngine::tellListener(Sp node, uint32_t socket_id, const InfoHash& h { auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6); try { - sendNodesValues(node->addr, TransId((char*)&socket_id, 4), nnodes.first, nnodes.second, values, query, ntoken); + sendNodesValues(node->getAddr(), TransId((char*)&socket_id, 4), nnodes.first, nnodes.second, values, query, ntoken); } catch (const std::overflow_error& e) { DHT_LOG.e("Can't send value: buffer not large enough !"); } @@ -205,8 +205,8 @@ NetworkEngine::requestStep(Sp sreq) } auto err = send((char*)req.msg.data(), req.msg.size(), - (node.reply_time >= now - UDP_REPLY_TIME) ? 0 : MSG_CONFIRM, - node.addr); + (node.getReplyTime() >= now - UDP_REPLY_TIME) ? 0 : MSG_CONFIRM, + node.getAddr()); if (err == ENETUNREACH || err == EHOSTUNREACH || err == EAFNOSUPPORT) @@ -298,7 +298,7 @@ void NetworkEngine::blacklistNode(const Sp& n) { n->setExpired(); - blacklist.emplace(n->addr); + blacklist.emplace(n->getAddr()); } bool @@ -414,7 +414,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro requests.erase(req_it); } else { node->received(now, req); - if (not node->is_client) + if (not node->isClient()) onNewNode(node, 1); throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find transaction", msg->id}; } @@ -422,7 +422,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro node->received(now, req); - if (not node->is_client) + if (not node->isClient()) onNewNode(node, 2); onReportedAddr(msg->id, msg->addr); @@ -466,7 +466,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro } } else { node->received(now, {}); - if (not node->is_client) + if (not node->isClient()) onNewNode(node, 1); try { switch (msg->type) { @@ -907,7 +907,7 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, std::vector n, ); sendRequest(req); if (not v.empty()) - sendValueParts(tid, v, n->addr); + sendValueParts(tid, v, n->getAddr()); ++out_stats.put; return req; } diff --git a/src/node_cache.cpp b/src/node_cache.cpp index f3a92334..65a1cd07 100644 --- a/src/node_cache.cpp +++ b/src/node_cache.cpp @@ -95,7 +95,7 @@ NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point if (not node) { node = std::make_shared(id, addr, client); it.first->second = node; - } else if (confirm || node->time < now - Node::NODE_EXPIRE_TIME) { + } else if (confirm or node->isOld(now)) { node->update(addr); } return node; diff --git a/src/search.h b/src/search.h index 9d7ae861..40a43e41 100644 --- a/src/search.h +++ b/src/search.h @@ -482,10 +482,10 @@ struct Dht::Search { * * @return true if a node has been removed, else false. */ - bool removeExpiredNode(time_point now) { + bool removeExpiredNode(const time_point& now) { for (auto e = nodes.cend(); e != nodes.cbegin();) { const Node& n = *(--e)->node; - if (n.isExpired() and n.time + Node::NODE_EXPIRE_TIME < now) { + if (n.isRemovable(now)) { //std::cout << "Removing expired node " << n.id << " from IPv" << (af==AF_INET?'4':'6') << " search " << id << std::endl; nodes.erase(e); return true; @@ -637,7 +637,7 @@ Dht::Search::insertNode(const Sp& snode, time_point now, const Blob& token step_time = time_point::min(); } n = nodes.insert(n, SearchNode(snode)); - node.time = now; + node.setTime(now); new_search_node = true; if (node.isExpired()) { if (not expired) From f2b9f8eeda55641a09fdd661c91db9c66ac6cced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 13 Oct 2017 01:06:47 +0200 Subject: [PATCH 033/156] dht: move bucket time update to routing table --- src/dht.cpp | 4 +--- src/routing_table.cpp | 10 +++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index b42db9ae..78a262ed 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -2077,10 +2077,8 @@ Dht::onError(Sp req, net::DhtProtocolException e) { } void -Dht::onReportedAddr(const InfoHash& id, const SockAddr& addr) +Dht::onReportedAddr(const InfoHash& /*id*/, const SockAddr& addr) { - const auto& b = buckets(addr.getFamily()).findBucket(id); - b->time = scheduler.time(); if (addr) reportedAddr(addr); } diff --git a/src/routing_table.cpp b/src/routing_table.cpp index 462d3c67..b0a95e9d 100644 --- a/src/routing_table.cpp +++ b/src/routing_table.cpp @@ -202,9 +202,13 @@ RoutingTable::split(const RoutingTable::iterator& b) } bool -RoutingTable::onNewNode(const Sp& node, int comfirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne) { +RoutingTable::onNewNode(const Sp& node, int confirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne) { auto b = findBucket(node->id); if (b == end()) return false; + + if (confirm == 2) + b->time = now; + for (auto& n : b->nodes) { if (n == node) return false; @@ -244,11 +248,11 @@ RoutingTable::onNewNode(const Sp& node, int comfirm, const time_point& now //DHT_LOG.d("Splitting from depth %u", depth(b)); b->sendCachedPing(ne); split(b); - return onNewNode(node, comfirm, now, myid, ne); + return onNewNode(node, confirm, now, myid, ne); } /* No space for this node. Cache it away for later. */ - if (comfirm or not b->cached) + if (confirm or not b->cached) b->cached = node; } else { /* Create a new node. */ From 1a54aff7bc662a733f98535036ec66490730f755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 13 Oct 2017 01:01:16 +0200 Subject: [PATCH 034/156] storage policies: use SockAddr API-breaking change to use const SockAddr& instead of sockaddr*. --- include/opendht/default_types.h | 6 +++--- include/opendht/securedht.h | 4 ++-- include/opendht/value.h | 9 +++++---- src/default_types.cpp | 10 +++++----- src/dht.cpp | 12 ++++++------ src/securedht.cpp | 8 ++++---- src/value.cpp | 2 +- 7 files changed, 26 insertions(+), 25 deletions(-) diff --git a/include/opendht/default_types.h b/include/opendht/default_types.h index 86582ad7..bef2f659 100644 --- a/include/opendht/default_types.h +++ b/include/opendht/default_types.h @@ -46,7 +46,7 @@ public: static Value::Filter getFilter() { return {}; } - static bool storePolicy(InfoHash key, std::shared_ptr& value, InfoHash from, const sockaddr* from_addr, socklen_t from_len); + static bool storePolicy(InfoHash key, std::shared_ptr& value, const InfoHash& from, const SockAddr&); static Value::Filter ServiceFilter(std::string s); @@ -210,7 +210,7 @@ public: addr.setPort(p); } - IpServiceAnnouncement(const sockaddr* sa, socklen_t sa_len) : addr(sa, sa_len) {} + IpServiceAnnouncement(const SockAddr& sa) : addr(sa) {} IpServiceAnnouncement(const Blob& b) { msgpack_unpack(unpackMsg(b).get()); @@ -246,7 +246,7 @@ public: return TYPE; } - static bool storePolicy(InfoHash, std::shared_ptr&, InfoHash, const sockaddr*, socklen_t); + static bool storePolicy(InfoHash, std::shared_ptr&, const InfoHash&, const SockAddr&); /** print value for debugging */ friend std::ostream& operator<< (std::ostream&, const IpServiceAnnouncement&); diff --git a/include/opendht/securedht.h b/include/opendht/securedht.h index acae729f..3c251941 100644 --- a/include/opendht/securedht.h +++ b/include/opendht/securedht.h @@ -158,7 +158,7 @@ private: const ValueType CERTIFICATE_TYPE = { 8, "Certificate", std::chrono::hours(24 * 7), // A certificate can only be stored at its public key ID. - [](InfoHash id, Sp& v, InfoHash, const sockaddr*, socklen_t) { + [](InfoHash id, Sp& v, const InfoHash&, const SockAddr&) { try { crypto::Certificate crt(v->data); // TODO check certificate signature @@ -166,7 +166,7 @@ const ValueType CERTIFICATE_TYPE = { } catch (const std::exception& e) {} return false; }, - [](InfoHash, const Sp& o, Sp& n, InfoHash, const sockaddr*, socklen_t) { + [](InfoHash, const Sp& o, Sp& n, const InfoHash&, const SockAddr&) { try { return crypto::Certificate(o->data).getPublicKey().getId() == crypto::Certificate(n->data).getPublicKey().getId(); } catch (const std::exception& e) {} diff --git a/include/opendht/value.h b/include/opendht/value.h index 407f480f..9fd10dfa 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -22,6 +22,7 @@ #include "infohash.h" #include "crypto.h" #include "utils.h" +#include "sockaddr.h" #include @@ -51,7 +52,7 @@ struct Query; * @param form_addr: network address of the incoming request. * @param from_len: network address lendth of the incoming request. */ -using StorePolicy = std::function& value, InfoHash from, const sockaddr* from_addr, socklen_t from_len)>; +using StorePolicy = std::function& value, const InfoHash& from, const SockAddr& addr)>; /** * An edition policy is applied once to every incoming value storage requests, @@ -67,15 +68,15 @@ using StorePolicy = std::function& val * @param form_addr: network address of the incoming request. * @param from_len: network address lendth of the incoming request. */ -using EditPolicy = std::function& old_val, std::shared_ptr& new_val, InfoHash from, const sockaddr* from_addr, socklen_t from_len)>; +using EditPolicy = std::function& old_val, std::shared_ptr& new_val, const InfoHash& from, const SockAddr& addr)>; static constexpr const size_t MAX_VALUE_SIZE {1024 * 64}; struct OPENDHT_PUBLIC ValueType { typedef uint16_t Id; - static bool DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr& v, InfoHash, const sockaddr*, socklen_t); - static bool DEFAULT_EDIT_POLICY(InfoHash, const std::shared_ptr&, std::shared_ptr&, InfoHash, const sockaddr*, socklen_t) { + static bool DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr& v, const InfoHash&, const SockAddr&); + static bool DEFAULT_EDIT_POLICY(InfoHash, const std::shared_ptr&, std::shared_ptr&, const InfoHash&, const SockAddr&) { return false; } diff --git a/src/default_types.cpp b/src/default_types.cpp index 7ec0e6c7..1e7bfd13 100644 --- a/src/default_types.cpp +++ b/src/default_types.cpp @@ -27,14 +27,14 @@ std::ostream& operator<< (std::ostream& s, const DhtMessage& v) } bool -DhtMessage::storePolicy(InfoHash h, std::shared_ptr& v, InfoHash f, const sockaddr* fa, socklen_t fas) +DhtMessage::storePolicy(InfoHash h, std::shared_ptr& v, const InfoHash& f, const SockAddr& sa) { try { auto msg = unpackMsg(v->data); if (msg.service.empty()) return false; } catch (const std::exception& e) {} - return ValueType::DEFAULT_STORE_POLICY(h, v, f, fa, fas); + return ValueType::DEFAULT_STORE_POLICY(h, v, f, sa); } Value::Filter @@ -66,17 +66,17 @@ std::ostream& operator<< (std::ostream& s, const IpServiceAnnouncement& v) } bool -IpServiceAnnouncement::storePolicy(InfoHash h, std::shared_ptr& v, InfoHash f, const sockaddr* from, socklen_t fromlen) +IpServiceAnnouncement::storePolicy(InfoHash h, std::shared_ptr& v, const InfoHash& f, const SockAddr& sa) { try { auto msg = unpackMsg(v->data); if (msg.getPort() == 0) return false; - IpServiceAnnouncement sa_addr {from, fromlen}; + IpServiceAnnouncement sa_addr {sa}; sa_addr.setPort(msg.getPort()); // argument v is modified (not the value). v = std::make_shared(IpServiceAnnouncement::TYPE, sa_addr, v->id); - return ValueType::DEFAULT_STORE_POLICY(h, v, f, from, fromlen); + return ValueType::DEFAULT_STORE_POLICY(h, v, f, sa); } catch (const std::exception& e) {} return false; } diff --git a/src/dht.cpp b/src/dht.cpp index 78a262ed..129b029d 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -1098,10 +1098,10 @@ Dht::getPut(const InfoHash& id, const Value::Id& vid) } return Sp {}; }; - auto v4 = find_value(searches4); - if (v4) return v4; - auto v6 = find_value(searches6); - if (v6) return v6; + if (auto v4 = find_value(searches4)) + return v4; + if (auto v6 = find_value(searches6)) + return v6; return {}; } @@ -2278,7 +2278,7 @@ Dht::onAnnounce(Sp n, DHT_LOG.d(hash, node.id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str()); } else { const auto& type = getType(lv->type); - if (type.editPolicy(hash, lv, vc, node.id, node.getAddr().get(), node.getAddr().getLength())) { + if (type.editPolicy(hash, lv, vc, node.id, node.getAddr())) { DHT_LOG.d(hash, node.id, "[store %s] editing %s", hash.toString().c_str(), vc->toString().c_str()); storageStore(hash, vc, created, node.getAddr()); @@ -2290,7 +2290,7 @@ Dht::onAnnounce(Sp n, } else { // Allow the value to be edited by the storage policy const auto& type = getType(vc->type); - if (type.storePolicy(hash, vc, node.id, node.getAddr().get(), node.getAddr().getLength())) { + if (type.storePolicy(hash, vc, node.id, node.getAddr())) { DHT_LOG.d(hash, node.id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str()); storageStore(hash, vc, created, node.getAddr()); } else { diff --git a/src/securedht.cpp b/src/securedht.cpp index d2c36008..ea15e125 100644 --- a/src/securedht.cpp +++ b/src/securedht.cpp @@ -76,7 +76,7 @@ SecureDht::~SecureDht() ValueType SecureDht::secureType(ValueType&& type) { - type.storePolicy = [this,type](InfoHash id, Sp& v, InfoHash nid, const sockaddr* a, socklen_t al) { + type.storePolicy = [this,type](InfoHash id, Sp& v, const InfoHash& nid, const SockAddr& a) { if (v->isSigned()) { if (!v->owner or !v->owner->checkSignature(v->getToSign(), v->signature)) { DHT_LOG.WARN("Signature verification failed"); @@ -85,11 +85,11 @@ SecureDht::secureType(ValueType&& type) else DHT_LOG.WARN("Signature verification succeeded"); } - return type.storePolicy(id, v, nid, a, al); + return type.storePolicy(id, v, nid, a); }; - type.editPolicy = [this,type](InfoHash id, const Sp& o, Sp& n, InfoHash nid, const sockaddr* a, socklen_t al) { + type.editPolicy = [this,type](InfoHash id, const Sp& o, Sp& n, const InfoHash& nid, const SockAddr& a) { if (!o->isSigned()) - return type.editPolicy(id, o, n, nid, a, al); + return type.editPolicy(id, o, n, nid, a); if (o->owner != n->owner) { DHT_LOG.WARN("Edition forbidden: owner changed."); return false; diff --git a/src/value.cpp b/src/value.cpp index 1195c172..90d1aaf3 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -71,7 +71,7 @@ std::ostream& operator<< (std::ostream& s, const Value& v) const ValueType ValueType::USER_DATA = {0, "User Data"}; bool -ValueType::DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr& v, InfoHash, const sockaddr*, socklen_t) +ValueType::DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr& v, const InfoHash&, const SockAddr&) { return v->size() <= MAX_VALUE_SIZE; } From 63e0c1266af3843771397827f03b8327ea65ec1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 13 Oct 2017 01:02:08 +0200 Subject: [PATCH 035/156] dht: use OpStatus in put Factorize shared pointers. --- src/dht.cpp | 104 ++++++++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 53 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index 129b029d..cd8c035f 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -877,6 +877,23 @@ Dht::cancelListen(const InfoHash& id, size_t token) return true; } +struct OpStatus { + struct Status { + bool done {false}; + bool ok {false}; + Status(bool done=false, bool ok=false) : done(done), ok(ok) {} + }; + Status status; + Status status4; + Status status6; +}; + +template +struct GetStatus : public OpStatus { + std::vector> values; + std::vector> nodes; +}; + void Dht::put(const InfoHash& id, Sp val, DoneCallback callback, time_point created, bool permanent) { @@ -890,54 +907,38 @@ Dht::put(const InfoHash& id, Sp val, DoneCallback callback, time_point cr DHT_LOG.d(id, "put: adding %s -> %s", id.toString().c_str(), val->toString().c_str()); - auto ok = std::make_shared(false); - auto done = std::make_shared(false); - auto done4 = std::make_shared(false); - auto done6 = std::make_shared(false); - auto donecb = [=](const std::vector>& nodes) { + auto op = std::make_shared(); + auto donecb = [callback](const std::vector>& nodes, OpStatus& op) { // Callback as soon as the value is announced on one of the available networks - if (callback && !*done && (*done4 && *done6)) { - callback(*ok, nodes); - *done = true; + if (callback and not op.status.done and (op.status4.done && op.status6.done)) { + callback(op.status4.ok or op.status6.ok, nodes); + op.status.done = true; } }; announce(id, AF_INET, val, [=](bool ok4, const std::vector>& nodes) { DHT_LOG.d(id, "Announce done IPv4 %d", ok4); - *done4 = true; - *ok |= ok4; - donecb(nodes); + auto& o = *op; + o.status4 = {true, ok4}; + donecb(nodes, o); }, created, permanent); announce(id, AF_INET6, val, [=](bool ok6, const std::vector>& nodes) { DHT_LOG.d(id, "Announce done IPv6 %d", ok6); - *done6 = true; - *ok |= ok6; - donecb(nodes); + auto& o = *op; + o.status6 = {true, ok6}; + donecb(nodes, o); }, created, permanent); } template -struct OpStatus { - struct Status { - bool done {false}; - bool ok {false}; - }; - Status status; - Status status4; - Status status6; - std::vector> values; - std::vector> nodes; -}; - -template -void doneCallbackWrapper(DoneCallback dcb, const std::vector>& nodes, Sp> op) { - if (op->status.done) +void doneCallbackWrapper(DoneCallback dcb, const std::vector>& nodes, GetStatus& op) { + if (op.status.done) return; - op->nodes.insert(op->nodes.end(), nodes.begin(), nodes.end()); - if (op->status.ok || (op->status4.done and op->status6.done)) { - bool ok = op->status.ok || op->status4.ok || op->status6.ok; - op->status.done = true; + op.nodes.insert(op.nodes.end(), nodes.begin(), nodes.end()); + if (op.status.ok or (op.status4.done and op.status6.done)) { + bool ok = op.status.ok or op.status4.ok or op.status6.ok; + op.status.done = true; if (dcb) - dcb(ok, op->nodes); + dcb(ok, op.nodes); } } @@ -946,17 +947,18 @@ bool callbackWrapper(Cb get_cb, DoneCallback done_cb, const std::vector>& values, std::function>(const std::vector>&)> add_values, - Sp> op) + Sp> o) { - if (op->status.done) + auto& op = *o; + if (op.status.done) return false; auto newvals = add_values(values); if (not newvals.empty()) { - op->status.ok = !get_cb(newvals); - op->values.insert(op->values.end(), newvals.begin(), newvals.end()); + op.status.ok = !get_cb(newvals); + op.values.insert(op.values.end(), newvals.begin(), newvals.end()); } doneCallbackWrapper(done_cb, {}, op); - return !op->status.ok; + return !op.status.ok; } void @@ -965,7 +967,7 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt scheduler.syncTime(); Query q {{}, where}; - auto op = std::make_shared>(); + auto op = std::make_shared>(); auto f = filter.chain(q.where.getFilter()); auto add_values = [op,f](const std::vector>& values) { @@ -988,22 +990,20 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt Dht::search(id, AF_INET, gcb, {}, [=](bool ok, const std::vector>& nodes) { //DHT_LOG_WARN("DHT done IPv4"); - op->status4.done = true; - op->status4.ok = ok; - doneCallbackWrapper(donecb, nodes, op); + op->status4 = {true, ok}; + doneCallbackWrapper(donecb, nodes, *op); }, f, q); Dht::search(id, AF_INET6, gcb, {}, [=](bool ok, const std::vector>& nodes) { //DHT_LOG_WARN("DHT done IPv6"); - op->status6.done = true; - op->status6.ok = ok; - doneCallbackWrapper(donecb, nodes, op); + op->status6 = {true, ok}; + doneCallbackWrapper(donecb, nodes, *op); }, f, q); } void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Query&& q) { scheduler.syncTime(); - auto op = std::make_shared>(); + auto op = std::make_shared>(); auto f = q.where.getFilter(); auto values = getLocal(id, f); @@ -1037,15 +1037,13 @@ void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Quer Dht::search(id, AF_INET, {}, qcb, [=](bool ok, const std::vector>& nodes) { //DHT_LOG_WARN("DHT done IPv4"); - op->status4.done = true; - op->status4.ok = ok; - doneCallbackWrapper(done_cb, nodes, op); + op->status4 = {true, ok}; + doneCallbackWrapper(done_cb, nodes, *op); }, f, q); Dht::search(id, AF_INET6, {}, qcb, [=](bool ok, const std::vector>& nodes) { //DHT_LOG_WARN("DHT done IPv6"); - op->status6.done = true; - op->status6.ok = ok; - doneCallbackWrapper(done_cb, nodes, op); + op->status6 = {true, ok}; + doneCallbackWrapper(done_cb, nodes, *op); }, f, q); } From 7df375fc75a18db7928b7c3496482c97330817f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 13 Oct 2017 14:27:53 +0200 Subject: [PATCH 036/156] NodeCache: fix invalid pointer access --- include/opendht/node_cache.h | 11 +++++----- src/node_cache.cpp | 39 +++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/include/opendht/node_cache.h b/include/opendht/node_cache.h index e5f90868..97e62cd1 100644 --- a/include/opendht/node_cache.h +++ b/include/opendht/node_cache.h @@ -26,9 +26,9 @@ namespace dht { struct NodeCache { - std::shared_ptr getNode(const InfoHash& id, sa_family_t family); - std::shared_ptr getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client=false); - std::vector> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count); + Sp getNode(const InfoHash& id, sa_family_t family); + Sp getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client=false); + std::vector> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const; /** * Reset the connectivity state of every node, @@ -40,11 +40,12 @@ struct NodeCache { private: class NodeMap : public std::map> { public: - std::shared_ptr getNode(const InfoHash& id); - std::shared_ptr getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client); + Sp getNode(const InfoHash& id); + Sp getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client); void clearBadNodes(); }; + const NodeMap& cache(sa_family_t af) const { return af == AF_INET ? cache_4 : cache_6; } NodeMap& cache(sa_family_t af) { return af == AF_INET ? cache_4 : cache_6; } NodeMap cache_4; NodeMap cache_6; diff --git a/src/node_cache.cpp b/src/node_cache.cpp index 65a1cd07..f821b29b 100644 --- a/src/node_cache.cpp +++ b/src/node_cache.cpp @@ -20,41 +20,44 @@ namespace dht { -std::shared_ptr +Sp NodeCache::getNode(const InfoHash& id, sa_family_t family) { return cache(family).getNode(id); } -std::shared_ptr +Sp NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) { if (id == zeroes) return std::make_shared(id, addr); return cache(addr.getFamily()).getNode(id, addr, now, confirm, client); } -std::vector> -NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) { +std::vector> +NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const +{ const auto& c = cache(sa_f); - auto it_p = c.lower_bound(id), - it_n = it_p; - std::vector> nodes; + std::vector> nodes; nodes.reserve(std::min(c.size(), count)); NodeMap::const_iterator it; + auto dec_it = [&c](NodeMap::const_iterator& it) { + auto ret = it; + it = (it == c.cbegin()) ? c.cend() : std::prev(it); + return ret; + }; - if (it_p != c.begin()) /* Create 2 separate iterator if we could */ - --it_p; + auto it_p = c.lower_bound(id), + it_n = it_p; + if (it_p != c.cend()) + dec_it(it_p); /* Create 2 separate iterator if we could */ - while (nodes.size() < count and (it_n != c.end() or it_p != c.end())) { + while (nodes.size() < count and (it_n != c.cend() or it_p != c.cend())) { /* If one of the iterator is at the end, then take the other one If they are both in middle of somewhere comapre both and take the closest to the id. */ - if (it_p == c.end()) it = it_n++; - else if (it_n == c.end()) it = it_p--; - else it = id.xorCmp(it_p->first, it_n->first) < 0 ? it_p-- : it_n++; - - if (it == c.begin()) - it_p = c.end(); + if (it_p == c.cend()) it = it_n++; + else if (it_n == c.cend()) it = dec_it(it_p); + else it = id.xorCmp(it_p->first, it_n->first) < 0 ? dec_it(it_p) : it_n++; if (auto n = it->second.lock()) if ( not n->isExpired() and not n->isClient() ) @@ -75,7 +78,7 @@ NodeCache::clearBadNodes(sa_family_t family) } } -std::shared_ptr +Sp NodeCache::NodeMap::getNode(const InfoHash& id) { auto wn = find(id); @@ -87,7 +90,7 @@ NodeCache::NodeMap::getNode(const InfoHash& id) return {}; } -std::shared_ptr +Sp NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) { auto it = emplace(id, std::weak_ptr{}); From 4856a6b4243debecdc59517164c77983d99bb9c9 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Sat, 22 Apr 2017 01:09:52 +0200 Subject: [PATCH 037/156] PublicKey: getId compatible with HASH_LEN = 32 --- src/crypto.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crypto.cpp b/src/crypto.cpp index dae67deb..8fb0fac4 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -529,7 +529,8 @@ PublicKey::getId() const return {}; InfoHash id; size_t sz = id.size(); - if (auto err = gnutls_pubkey_get_key_id(pk, 0, id.data(), &sz)) + constexpr int flags = (id.size() == 32) ? GNUTLS_KEYID_USE_SHA256 : 0; + if (auto err = gnutls_pubkey_get_key_id(pk, flags, id.data(), &sz)) throw CryptoException(std::string("Can't get public key ID: ") + gnutls_strerror(err)); if (sz != id.size()) throw CryptoException("Can't get public key ID: wrong output length."); From 9d4e6e09842d765d6cf14bc51bfda3c55ed98676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Sat, 14 Oct 2017 14:37:05 +0200 Subject: [PATCH 038/156] rate limiter: use deterministic maintenance Using deterministic maintenance sets an upper bound to the size of the address map. --- include/opendht/network_engine.h | 5 ++--- include/opendht/rate_limiter.h | 8 ++++---- src/network_engine.cpp | 7 +++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/include/opendht/network_engine.h b/include/opendht/network_engine.h index 2e309927..1fc5f020 100644 --- a/include/opendht/network_engine.h +++ b/include/opendht/network_engine.h @@ -34,7 +34,6 @@ #include #include #include -#include #include namespace dht { @@ -417,7 +416,7 @@ private: /*************** * Constants * ***************/ - static constexpr long unsigned MAX_REQUESTS_PER_SEC {1600}; + static constexpr size_t MAX_REQUESTS_PER_SEC {1600}; /* the length of a node info buffer in ipv4 format */ static const constexpr size_t NODE4_INFO_BUF_LEN {HASH_LEN + sizeof(in_addr) + sizeof(in_port_t)}; /* the length of a node info buffer in ipv6 format */ @@ -437,7 +436,6 @@ private: static constexpr size_t MAX_PACKET_VALUE_SIZE {600}; static const std::string my_v; - static std::mt19937 rd_device; void process(std::unique_ptr&&, const SockAddr& from); @@ -518,6 +516,7 @@ private: using IpLimiterMap = std::map; IpLimiterMap address_rate_limiter {}; RateLimiter rate_limiter {}; + size_t limiter_maintenance {0}; // requests handling std::map> requests {}; diff --git a/include/opendht/rate_limiter.h b/include/opendht/rate_limiter.h index 504c8934..061b36a5 100644 --- a/include/opendht/rate_limiter.h +++ b/include/opendht/rate_limiter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Savoir-faire Linux Inc. + * Copyright (C) 2017 Savoir-faire Linux Inc. * Author : Adrien Béraud * * This program is free software; you can redistribute it and/or modify @@ -26,10 +26,10 @@ namespace dht { template class RateLimiter { public: - /** Clear outdated records and return current quota usage */ + /** Clear outdated records and return current quota usage */ size_t maintain(const time_point& now) { - using namespace std::chrono; - while (not records.empty() and duration_cast(now - records.front()) >= seconds(Perdiod)) + auto limit = now - std::chrono::seconds(Perdiod); + while (not records.empty() and records.front() < limit) records.pop(); return records.size(); } diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 8843ee3f..4986ae82 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -54,7 +54,6 @@ constexpr std::chrono::seconds NetworkEngine::RX_TIMEOUT; const std::string NetworkEngine::my_v {"RNG1"}; const constexpr uint16_t TransId::INVALID; -std::mt19937 NetworkEngine::rd_device {dht::crypto::random_device{}()}; const TransPrefix TransPrefix::PING = {"pn"}; const TransPrefix TransPrefix::FIND_NODE = {"fn"}; @@ -249,15 +248,15 @@ NetworkEngine::rateLimit(const SockAddr& addr) { const auto& now = scheduler.time(); - // occasional IP limiter maintenance - std::bernoulli_distribution rand_trial(1./128.); - if (rand_trial(rd_device)) { + // occasional IP limiter maintenance (a few times every second at max rate) + if (limiter_maintenance++ == MAX_REQUESTS_PER_SEC/8) { for (auto it = address_rate_limiter.begin(); it != address_rate_limiter.end();) { if (it->second.maintain(now) == 0) address_rate_limiter.erase(it++); else ++it; } + limiter_maintenance = 0; } auto it = address_rate_limiter.emplace(addr, IpLimiter{}); From be2b9fd7dc3c2fe97a235a176ac439571f9eacca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Sun, 15 Oct 2017 15:52:30 +0200 Subject: [PATCH 039/156] NodeCache: handle end() lower_bound for non-empty map --- src/node_cache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node_cache.cpp b/src/node_cache.cpp index f821b29b..68b6441b 100644 --- a/src/node_cache.cpp +++ b/src/node_cache.cpp @@ -48,7 +48,7 @@ NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) co auto it_p = c.lower_bound(id), it_n = it_p; - if (it_p != c.cend()) + if (not c.empty()) dec_it(it_p); /* Create 2 separate iterator if we could */ while (nodes.size() < count and (it_n != c.cend() or it_p != c.cend())) { From 2dd94fbfe85901d849393bddd799839a7598f53e Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Mon, 16 Oct 2017 01:26:40 +0200 Subject: [PATCH 040/156] cmake: use CMAKE_THREAD_LIBS_INIT as library --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index da8731da..32d39b1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,7 @@ endif () # Build flags set (CMAKE_CXX_STANDARD 11) set (CMAKE_CXX_STANDARD_REQUIRED on) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_THREAD_LIBS_INIT} -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden") if (OPENDHT_SANITIZE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fstack-protector-strong") endif () @@ -177,7 +177,7 @@ if (OPENDHT_STATIC) target_link_libraries(opendht-static ${argon2_LIBRARIES}) target_include_directories(opendht-static SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - target_link_libraries(opendht-static ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) + target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () @@ -198,7 +198,7 @@ if (OPENDHT_SHARED) target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES}) target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - target_link_libraries(opendht PRIVATE ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) + target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () From 6bc2f19a36771b14e41449560d954a247062ab37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 16 Oct 2017 16:55:26 +0200 Subject: [PATCH 041/156] dht: log total number of nodes in onGetValuesDone --- src/dht.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dht.cpp b/src/dht.cpp index cd8c035f..4a3485cf 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -2136,7 +2136,7 @@ void Dht::onGetValuesDone(const Sp& node, } DHT_LOG.d(sr->id, "[search %s] [node %s] got reply to 'get' with %u nodes", - sr->id.toString().c_str(), node->toString().c_str(), a.nodes4.size()); + sr->id.toString().c_str(), node->toString().c_str(), a.nodes4.size()+a.nodes6.size()); if (not a.ntoken.empty()) { if (not a.values.empty() or not a.fields.empty()) { From 488aa771a956fff01f079a2e1cedc096ccd27278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 16 Oct 2017 16:56:13 +0200 Subject: [PATCH 042/156] Node: don't expire existing requests --- src/node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node.cpp b/src/node.cpp index 26cdb8de..7974b36e 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -79,7 +79,7 @@ void Node::requested(const Sp& req) { auto e = requests_.emplace(req->getTid(), req); - if (not e.second) { + if (not e.second and req != e.first->second) { // Should not happen ! // Try to handle this scenario as well as we can e.first->second->setExpired(); From 63013bf3747d68a81af0469acb1a005fd440e6e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 17 Oct 2017 03:17:52 +0200 Subject: [PATCH 043/156] SearchNode: prevent copy, check request access --- src/search.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/search.h b/src/search.h index 40a43e41..e5aeee82 100644 --- a/src/search.h +++ b/src/search.h @@ -76,6 +76,11 @@ struct Dht::SearchNode { node is a new candidate for inclusion. */ SearchNode() : node() {} + SearchNode(const SearchNode&) = delete; + SearchNode(SearchNode&&) = default; + SearchNode& operator=(const SearchNode&) = delete; + SearchNode& operator=(SearchNode&&) = default; + SearchNode(const Sp& node) : node(node) {} ~SearchNode() { if (node) { @@ -186,6 +191,7 @@ struct Dht::SearchNode { node->cancelRequest(status.second); } } + getStatus.clear(); } /** @@ -227,10 +233,11 @@ struct Dht::SearchNode { void cancelAnnounce() { for (const auto& status : acked) { const auto& req = status.second.first; - if (req->pending()) { + if (req and req->pending()) { node->cancelRequest(req); } } + acked.clear(); } bool isListening(time_point now) const { From 376486e64fbd3b1774dd272ec33927da46063496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 17 Oct 2017 02:22:16 +0200 Subject: [PATCH 044/156] node: use full range of socket IDs --- include/opendht/net.h | 31 ++++++--------------- include/opendht/node.h | 20 ++++++++++---- src/dht.cpp | 62 +++++++++++++++++++++--------------------- src/network_engine.cpp | 17 ++++++------ src/node.cpp | 40 +++++++++++++-------------- src/parsed_message.h | 5 ++-- src/request.h | 24 +++------------- src/utils.cpp | 15 ++++++++++ 8 files changed, 104 insertions(+), 110 deletions(-) diff --git a/include/opendht/net.h b/include/opendht/net.h index 180c4cf9..fd680e51 100644 --- a/include/opendht/net.h +++ b/include/opendht/net.h @@ -42,41 +42,26 @@ struct TransPrefix : public std::array { struct TransId final : public std::array { static const constexpr uint16_t INVALID {0}; - TransId() { std::fill_n(begin(), 4, 0); } + TransId() { std::fill(begin(), end(), 0); } TransId(const std::array& o) { std::copy(o.begin(), o.end(), begin()); } TransId(const TransPrefix prefix, uint16_t seqno = 0) { std::copy_n(prefix.begin(), prefix.size(), begin()); *reinterpret_cast(data()+prefix.size()) = seqno; } - - TransId(const char* q, size_t l) : array() { - if (l > 4) { - length = 0; - } else { - std::copy_n(q, l, begin()); - length = l; - } - } - - uint16_t getTid() const { - return *reinterpret_cast(&(*this)[2]); + TransId(uint32_t id) { + *reinterpret_cast(data()) = htonl(id); } uint32_t toInt() const { - return *reinterpret_cast(&(*this)[0]); + return ntohl(*reinterpret_cast(&(*this)[0])); } - bool matches(const TransPrefix prefix, uint16_t* tid = nullptr) const { - if (std::equal(begin(), begin()+2, prefix.begin())) { - if (tid) - *tid = getTid(); - return true; - } else - return false; + bool matches(const TransPrefix prefix) const { + return std::equal(begin(), begin()+2, prefix.begin()); } - - unsigned length {4}; }; +TransId unpackTid(msgpack::object& o); + } /* namespace net */ } /* dht */ diff --git a/include/opendht/node.h b/include/opendht/node.h index 22e7e562..1f6efda9 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -34,9 +34,17 @@ namespace net { struct Request; struct Socket; struct RequestAnswer; -using SocketCb = std::function&, RequestAnswer&&)>; } /* namespace net */ +using SocketCb = std::function&, net::RequestAnswer&&)>; +using SocketId = uint32_t; +struct Socket { + Socket() {} + Socket(SocketCb&& on_receive) : + on_receive(std::move(on_receive)) {} + SocketCb on_receive {}; +}; + struct Node { const InfoHash id; @@ -108,16 +116,16 @@ struct Node { * * @return the socket. */ - Sp openSocket(const net::TransId& id, net::SocketCb&& cb); + SocketId openSocket(SocketCb&& cb); - Sp getSocket(const net::TransId& id) const; + Socket* getSocket(SocketId id); /** * Closes a socket so that no further data will be red on that socket. * * @param socket The socket to close. */ - void closeSocket(const Sp& socket); + void closeSocket(SocketId id); /** * Resets the state of the node so it's not expired anymore. @@ -156,11 +164,11 @@ private: time_point reply_time {time_point::min()}; /* time of last correct reply received */ unsigned auth_errors {0}; bool expired_ {false}; - uint16_t transaction_id {1}; + SocketId transaction_id; using TransactionDist = std::uniform_int_distribution; std::map> requests_ {}; - std::map> sockets_ {}; + std::map sockets_; }; } diff --git a/src/dht.cpp b/src/dht.cpp index 4a3485cf..b79f4ecf 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -540,40 +540,40 @@ Dht::searchStep(Sp sr) continue; for (const auto& l : sr->listeners) { const auto& query = l.second.query; - if (n.getListenTime(query) <= now) { - DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'", - sr->id.toString().c_str(), n.node->toString().c_str()); + if (n.getListenTime(query) > now) + continue; + DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'", + sr->id.toString().c_str(), n.node->toString().c_str()); - const auto& r = n.listenStatus.find(query); - auto prev_req = r != n.listenStatus.end() ? r->second : nullptr; + const auto& r = n.listenStatus.find(query); + auto prev_req = r != n.listenStatus.end() ? r->second : nullptr; - std::weak_ptr ws = sr; - n.listenStatus[query] = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req, - [this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable - { /* on done */ - if (auto sr = ws.lock()) { - onListenDone(req.node, answer, sr); - scheduler.edit(sr->nextSearchStep, scheduler.time()); - } - }, - [this,ws,query](const net::Request& req, bool over) mutable - { /* on expired */ - if (auto sr = ws.lock()) { - scheduler.edit(sr->nextSearchStep, scheduler.time()); - if (over) - if (auto sn = sr->getNode(req.node)) - sn->listenStatus.erase(query); - } - }, - [this,ws,query](const Sp& node, net::RequestAnswer&& answer) mutable - { /* on new values */ - if (auto sr = ws.lock()) { - onGetValuesDone(node, answer, sr, query); - scheduler.edit(sr->nextSearchStep, scheduler.time()); - } + std::weak_ptr ws = sr; + n.listenStatus[query] = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req, + [this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable + { /* on done */ + if (auto sr = ws.lock()) { + onListenDone(req.node, answer, sr); + scheduler.edit(sr->nextSearchStep, scheduler.time()); } - ); - } + }, + [this,ws,query](const net::Request& req, bool over) mutable + { /* on expired */ + if (auto sr = ws.lock()) { + scheduler.edit(sr->nextSearchStep, scheduler.time()); + if (over) + if (auto sn = sr->getNode(req.node)) + sn->listenStatus.erase(query); + } + }, + [this,ws,query](const Sp& node, net::RequestAnswer&& answer) mutable + { /* on new values */ + if (auto sr = ws.lock()) { + onGetValuesDone(node, answer, sr, query); + scheduler.edit(sr->nextSearchStep, scheduler.time()); + } + } + ); } if (not n.candidate and ++i == LISTEN_NODES) break; diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 4986ae82..04fefd25 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -153,7 +153,7 @@ NetworkEngine::tellListener(Sp node, uint32_t socket_id, const InfoHash& h { auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6); try { - sendNodesValues(node->getAddr(), TransId((char*)&socket_id, 4), nnodes.first, nnodes.second, values, query, ntoken); + sendNodesValues(node->getAddr(), TransId(socket_id), nnodes.first, nnodes.second, values, query, ntoken); } catch (const std::overflow_error& e) { DHT_LOG.e("Can't send value: buffer not large enough !"); } @@ -401,7 +401,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro auto node = cache.getNode(msg->id, from, now, true, msg->is_client); if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { - auto rsocket = node->getSocket(msg->tid); + auto rsocket = node->getSocket(msg->tid.toInt()); auto req = node->getRequest(msg->tid); /* either response for a request or data for an opened socket */ @@ -954,20 +954,21 @@ NetworkEngine::sendListen(Sp n, RequestExpiredCb&& on_expired, SocketCb&& socket_cb) { - Sp socket; - auto tid = TransId { TransPrefix::LISTEN, previous ? previous->tid.getTid() : n->getNewTid() }; + uint32_t socket; + auto tid = TransId { TransPrefix::LISTEN, n->getNewTid() }; if (previous and previous->node == n) { - socket = previous->socket; + socket = previous->getSocket(); } else { if (previous) DHT_LOG.e(hash, "[node %s] trying refresh listen contract with wrong node", previous->node->toString().c_str()); - socket = n->openSocket(TransPrefix::GET_VALUES, std::move(socket_cb)); + socket = n->openSocket(std::move(socket_cb)); } if (not socket) { DHT_LOG.e(hash, "[node %s] unable to get a valid socket for listen. Aborting listen", n->toString().c_str()); return {}; } + TransId sid(socket); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); @@ -978,8 +979,8 @@ NetworkEngine::sendListen(Sp n, pk.pack(std::string("id")); pk.pack(myid); pk.pack(std::string("h")); pk.pack(hash); pk.pack(std::string("token")); packToken(pk, token); - pk.pack(std::string("sid")); pk.pack_bin(socket->id.size()); - pk.pack_bin_body((const char*)socket->id.data(), socket->id.size()); + pk.pack(std::string("sid")); pk.pack_bin(sid.size()); + pk.pack_bin_body((const char*)sid.data(), sid.size()); if (has_query) { pk.pack(std::string("q")); pk.pack(query); } diff --git a/src/node.cpp b/src/node.cpp index 7974b36e..ba8f4cb9 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -32,10 +32,10 @@ constexpr std::chrono::seconds Node::MAX_RESPONSE_TIME; Node::Node(const InfoHash& id, const SockAddr& addr, bool client) -: id(id), addr(addr), is_client(client) +: id(id), addr(addr), is_client(client), sockets_() { crypto::random_device rd; - transaction_id = TransactionDist{1}(rd); + transaction_id = std::uniform_int_distribution{1}(rd); } /* This is our definition of a known-good node. */ @@ -112,7 +112,7 @@ Node::cancelRequest(const Sp& req) { if (req) { req->cancel(); - closeSocket(req->getSocket()); + closeSocket(req->closeSocket()); requests_.erase(req->getTid()); } } @@ -128,31 +128,31 @@ Node::setExpired() sockets_.clear(); } - -Sp -Node::openSocket(const net::TransId& tid, net::SocketCb&& cb) +SocketId +Node::openSocket(SocketCb&& cb) { - auto s = sockets_.emplace(tid, std::make_shared(tid, cb)); - //if (not s.second) - // DHT_LOG.e(id, "[node %s] socket (tid: %d) already opened!", id.toString().c_str(), tid.toInt()); - //else - // DHT_LOG.w("Opened socket (tid: %d), %lu opened", s.first->second->id, sockets_.size()); - return s.first->second; + if (++transaction_id == 0) + transaction_id = 1; + + auto sock = Socket(std::move(cb)); + auto s = sockets_.emplace(transaction_id, std::move(sock)); + if (not s.second) + s.first->second = std::move(sock); + return transaction_id; } - -Sp -Node::getSocket(const net::TransId& tid) const +Socket* +Node::getSocket(SocketId id) { - auto it = sockets_.find(tid); - return it == sockets_.end() ? nullptr : it->second; + auto it = sockets_.find(id); + return it == sockets_.end() ? nullptr : &it->second; } void -Node::closeSocket(const Sp& socket) +Node::closeSocket(SocketId id) { - if (socket) { - sockets_.erase(socket->id); + if (id) { + sockets_.erase(id); //DHT_LOG.w("Closing socket (tid: %d), %lu remaining", socket->id, sockets_.size()); } } diff --git a/src/parsed_message.h b/src/parsed_message.h index f2f691d6..1164342e 100644 --- a/src/parsed_message.h +++ b/src/parsed_message.h @@ -19,6 +19,7 @@ #include "infohash.h" #include "sockaddr.h" +#include "net.h" #include @@ -131,7 +132,7 @@ ParsedMessage::msgpack_unpack(msgpack::object msg) auto v = findMapValue(msg, "p"); if (auto t = findMapValue(msg, "t")) - tid = {t->as>()}; + tid = unpackTid(*t); if (auto rv = findMapValue(msg, "v")) ua = rv->as(); @@ -198,7 +199,7 @@ ParsedMessage::msgpack_unpack(msgpack::object msg) } if (auto t = findMapValue(req, "sid")) - socket_id = {t->as>()}; + socket_id = unpackTid(*t); if (auto rid = findMapValue(req, "id")) id = {*rid}; diff --git a/src/request.h b/src/request.h index d75554f1..0b222eba 100644 --- a/src/request.h +++ b/src/request.h @@ -28,23 +28,6 @@ namespace net { class NetworkEngine; struct ParsedMessage; -struct RequestAnswer; -using SocketCb = std::function&, RequestAnswer&&)>; - -/** - * Open route to a node for continous incoming packets. - * A socket lets a remote node send us continuous packets treated using a - * given callback. This is intended to provide an easy management of - * specific updates nodes can send. For e.g, this is used in the case of the - * "listen" operation for treating updates a node has for a given storage. - */ -struct Socket { - Socket() {} - Socket(TransId id, SocketCb on_receive) : - id(id), on_receive(on_receive) {} - TransId id; - SocketCb on_receive {}; -}; /*! * @class Request @@ -81,12 +64,13 @@ struct Request { Blob&& msg, std::function on_done, std::function on_expired, - Sp socket = {}) : + uint32_t socket = 0) : node(node), on_done(on_done), on_expired(on_expired), tid(tid), msg(std::move(msg)), socket(socket) { } TransId getTid() const { return tid; } - const Sp& getSocket() const { return socket; } + uint32_t getSocket() { return socket; } + uint32_t closeSocket() { auto ret = socket; socket = 0; return ret; } void setExpired() { if (pending()) { @@ -134,7 +118,7 @@ private: const TransId tid; /* the request id. */ Blob msg {}; /* the serialized message. */ - Sp socket; /* the socket used for further reponses. */ + uint32_t socket; /* the socket used for further reponses. */ }; } /* namespace net */ diff --git a/src/utils.cpp b/src/utils.cpp index 87826e1f..cb2fd5b3 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -19,6 +19,7 @@ #include "utils.h" #include "sockaddr.h" #include "default_types.h" +#include "net.h" /* An IPv4 equivalent to IN6_IS_ADDR_UNSPECIFIED */ #ifndef IN_IS_ADDR_UNSPECIFIED @@ -160,4 +161,18 @@ findMapValue(msgpack::object& map, const std::string& key) { return nullptr; } +namespace net { + +TransId +unpackTid(msgpack::object& o) { + switch (o.type) { + case msgpack::type::POSITIVE_INTEGER: + return o.as(); + default: + return o.as>(); + } +} + +} + } From 775eb2e59908648ad561b5db505273a7b73ae87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 17 Oct 2017 18:29:16 +0200 Subject: [PATCH 045/156] update to version 1.4.0 * improved Doxygen documentation * cmake: add support to build documentation * cmake: improve build on macOS * API change: storage policies, getPublicAddress now use SockAddr * NodeExport: added msgpack serialization * fixed issue that could cause occasional request failures * minor bug fixes * minor performance improvements --- CMakeLists.txt | 2 +- configure.ac | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 32d39b1f..0cbb6eb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required (VERSION 3.1) project (opendht) set (opendht_VERSION_MAJOR 1) -set (opendht_VERSION_MINOR 3.6) +set (opendht_VERSION_MINOR 4.0) set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR}) set (PACKAGE_VERSION ${opendht_VERSION}) set (VERSION "${opendht_VERSION}") diff --git a/configure.ac b/configure.ac index 3a4b8eaf..31a628b7 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl define macros m4_define([opendht_major_version], 1) -m4_define([opendht_minor_version], 3) -m4_define([opendht_patch_version], 6) +m4_define([opendht_minor_version], 4) +m4_define([opendht_patch_version], 0) m4_define([opendht_version], [opendht_major_version.opendht_minor_version.opendht_patch_version]) From cf6447dc8018f90ac064d1c6cab1a005eaf4dad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 17 Oct 2017 19:20:38 +0200 Subject: [PATCH 046/156] NetworkEngine: use proper type for constant --- src/network_engine.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 04fefd25..26836f14 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -61,7 +61,7 @@ const TransPrefix TransPrefix::GET_VALUES = {"gt"}; const TransPrefix TransPrefix::ANNOUNCE_VALUES = {"pt"}; const TransPrefix TransPrefix::REFRESH = {"rf"}; const TransPrefix TransPrefix::LISTEN = {"lt"}; -constexpr long unsigned NetworkEngine::MAX_REQUESTS_PER_SEC; +constexpr size_t NetworkEngine::MAX_REQUESTS_PER_SEC; static const uint8_t v4prefix[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0, 0, 0, 0 From 207675a181478c209f025416119e1ce651e48044 Mon Sep 17 00:00:00 2001 From: atraczyk Date: Wed, 18 Oct 2017 17:50:19 -0400 Subject: [PATCH 047/156] remove vs2015 based constexpr error, update vs project files --- MSVC/opendht.vcxproj | 4 ++++ src/crypto.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/MSVC/opendht.vcxproj b/MSVC/opendht.vcxproj index ce94bdd5..cfa9fcc9 100644 --- a/MSVC/opendht.vcxproj +++ b/MSVC/opendht.vcxproj @@ -39,6 +39,7 @@ + @@ -46,6 +47,7 @@ + @@ -55,9 +57,11 @@ + + diff --git a/src/crypto.cpp b/src/crypto.cpp index 8fb0fac4..6c029d56 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -529,7 +529,7 @@ PublicKey::getId() const return {}; InfoHash id; size_t sz = id.size(); - constexpr int flags = (id.size() == 32) ? GNUTLS_KEYID_USE_SHA256 : 0; + const int flags = (id.size() == 32) ? GNUTLS_KEYID_USE_SHA256 : 0; if (auto err = gnutls_pubkey_get_key_id(pk, flags, id.data(), &sz)) throw CryptoException(std::string("Can't get public key ID: ") + gnutls_strerror(err)); if (sz != id.size()) From 476fbb11609f8df5b727a492b4bf10bcfad5045a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 20 Oct 2017 13:45:46 +0200 Subject: [PATCH 048/156] network: use full 32-bits of transaction IDs We used to use the first 16 bits for the message type. --- CMakeLists.txt | 2 +- MSVC/opendht.vcxproj | 2 +- include/opendht/net.h | 67 ------------------- include/opendht/network_engine.h | 24 +++---- include/opendht/node.h | 21 +++--- src/Makefile.am | 2 +- src/net.h | 37 +++++++++++ src/network_engine.cpp | 110 ++++++++++++++++++------------- src/node.cpp | 10 +-- src/parsed_message.h | 23 +++---- src/request.h | 18 ++--- src/utils.cpp | 15 ----- 12 files changed, 151 insertions(+), 180 deletions(-) delete mode 100644 include/opendht/net.h create mode 100644 src/net.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cbb6eb6..03d21b71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,7 @@ list (APPEND opendht_SOURCES src/storage.h src/listener.h src/search.h + src/net.h src/parsed_message.h src/request.h src/callbacks.cpp @@ -134,7 +135,6 @@ list (APPEND opendht_HEADERS include/opendht/routing_table.h include/opendht/node_cache.h include/opendht/network_engine.h - include/opendht/net.h include/opendht/scheduler.h include/opendht/rate_limiter.h include/opendht/securedht.h diff --git a/MSVC/opendht.vcxproj b/MSVC/opendht.vcxproj index cfa9fcc9..4c494bf5 100644 --- a/MSVC/opendht.vcxproj +++ b/MSVC/opendht.vcxproj @@ -47,7 +47,6 @@ - @@ -60,6 +59,7 @@ + diff --git a/include/opendht/net.h b/include/opendht/net.h deleted file mode 100644 index fd680e51..00000000 --- a/include/opendht/net.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2014-2017 Savoir-faire Linux Inc. - * Author(s) : Adrien Béraud - * Simon Désaulniers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include -#include - -namespace dht { -namespace net { - -struct TransPrefix : public std::array { - TransPrefix(const std::string& str) : std::array({{(uint8_t)str[0], (uint8_t)str[1]}}) {} - static const TransPrefix PING; - static const TransPrefix FIND_NODE; - static const TransPrefix GET_VALUES; - static const TransPrefix ANNOUNCE_VALUES; - static const TransPrefix REFRESH; - static const TransPrefix LISTEN; -}; - -/* Transaction-ids are 4-bytes long, with the first two bytes identifying - * the kind of request, and the remaining two a sequence number in - * host order. - */ -struct TransId final : public std::array { - static const constexpr uint16_t INVALID {0}; - - TransId() { std::fill(begin(), end(), 0); } - TransId(const std::array& o) { std::copy(o.begin(), o.end(), begin()); } - TransId(const TransPrefix prefix, uint16_t seqno = 0) { - std::copy_n(prefix.begin(), prefix.size(), begin()); - *reinterpret_cast(data()+prefix.size()) = seqno; - } - TransId(uint32_t id) { - *reinterpret_cast(data()) = htonl(id); - } - - uint32_t toInt() const { - return ntohl(*reinterpret_cast(&(*this)[0])); - } - - bool matches(const TransPrefix prefix) const { - return std::equal(begin(), begin()+2, prefix.begin()); - } -}; - -TransId unpackTid(msgpack::object& o); - -} /* namespace net */ -} /* dht */ diff --git a/include/opendht/network_engine.h b/include/opendht/network_engine.h index 1fc5f020..ba9a06ef 100644 --- a/include/opendht/network_engine.h +++ b/include/opendht/network_engine.h @@ -27,7 +27,6 @@ #include "utils.h" #include "rng.h" #include "rate_limiter.h" -#include "net.h" #include #include @@ -41,6 +40,7 @@ namespace net { struct Request; struct Socket; +struct TransId; #ifndef MSG_CONFIRM #define MSG_CONFIRM 0 @@ -169,7 +169,7 @@ private: std::function, const InfoHash&, const Blob&, - uint32_t, + Tid, const Query&)> onListen {}; /** * Called on announce request. @@ -231,7 +231,7 @@ public: * @param nodes6 The ipv6 closest nodes. * @param values The values to send. */ - void tellListener(Sp n, uint32_t socket_id, const InfoHash& hash, want_t want, const Blob& ntoken, + void tellListener(Sp n, Tid socket_id, const InfoHash& hash, want_t want, const Blob& ntoken, std::vector>&& nodes, std::vector>&& nodes6, std::vector>&& values, const Query& q); @@ -465,18 +465,18 @@ private: // basic wrapper for socket sendto function int send(const char *buf, size_t len, int flags, const SockAddr& addr); - void sendValueParts(TransId tid, const std::vector& svals, const SockAddr& addr); + void sendValueParts(const TransId& tid, const std::vector& svals, const SockAddr& addr); std::vector packValueHeader(msgpack::sbuffer&, const std::vector>&); - void maintainRxBuffer(const TransId& tid); + void maintainRxBuffer(Tid tid); /************* * Answers * *************/ /* answer to a ping request */ - void sendPong(const SockAddr& addr, TransId tid); + void sendPong(const SockAddr& addr, Tid tid); /* answer to findnodes/getvalues request */ void sendNodesValues(const SockAddr& addr, - TransId tid, + Tid tid, const Blob& nodes, const Blob& nodes6, const std::vector>& st, @@ -490,12 +490,12 @@ private: std::vector>& nodes, std::vector>& nodes6); /* answer to a listen request */ - void sendListenConfirmation(const SockAddr& addr, TransId tid); + void sendListenConfirmation(const SockAddr& addr, Tid tid); /* answer to put request */ - void sendValueAnnounced(const SockAddr& addr, TransId, Value::Id); + void sendValueAnnounced(const SockAddr& addr, Tid, Value::Id); /* answer in case of error */ void sendError(const SockAddr& addr, - TransId tid, + Tid tid, uint16_t code, const std::string& message, bool include_id=false); @@ -519,8 +519,8 @@ private: size_t limiter_maintenance {0}; // requests handling - std::map> requests {}; - std::map partial_messages; + std::map> requests {}; + std::map partial_messages; MessageStats in_stats {}, out_stats {}; std::set blacklist {}; diff --git a/include/opendht/node.h b/include/opendht/node.h index 1f6efda9..394b1156 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -22,7 +22,6 @@ #include "infohash.h" // includes socket structures #include "utils.h" #include "sockaddr.h" -#include "net.h" #include #include @@ -36,8 +35,8 @@ struct Socket; struct RequestAnswer; } /* namespace net */ +using Tid = uint32_t; using SocketCb = std::function&, net::RequestAnswer&&)>; -using SocketId = uint32_t; struct Socket { Socket() {} Socket(SocketCb&& on_receive) : @@ -102,7 +101,7 @@ struct Node { void requested(const Sp& req); void received(time_point now, const Sp& req); - Sp getRequest(const net::TransId& tid); + Sp getRequest(Tid tid); void cancelRequest(const Sp& req); void setExpired(); @@ -116,16 +115,16 @@ struct Node { * * @return the socket. */ - SocketId openSocket(SocketCb&& cb); + Tid openSocket(SocketCb&& cb); - Socket* getSocket(SocketId id); + Socket* getSocket(Tid id); /** * Closes a socket so that no further data will be red on that socket. * * @param socket The socket to close. */ - void closeSocket(SocketId id); + void closeSocket(Tid id); /** * Resets the state of the node so it's not expired anymore. @@ -137,9 +136,9 @@ struct Node { * * @return the new id. */ - uint16_t getNewTid() { + Tid getNewTid() { ++transaction_id; - return transaction_id == net::TransId::INVALID ? ++transaction_id : transaction_id; + return transaction_id ? ++transaction_id : transaction_id; } std::string toString() const; @@ -164,11 +163,11 @@ private: time_point reply_time {time_point::min()}; /* time of last correct reply received */ unsigned auth_errors {0}; bool expired_ {false}; - SocketId transaction_id; + Tid transaction_id; using TransactionDist = std::uniform_int_distribution; - std::map> requests_ {}; - std::map sockets_; + std::map> requests_ {}; + std::map sockets_; }; } diff --git a/src/Makefile.am b/src/Makefile.am index 634f2d88..f4614795 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,6 +9,7 @@ libopendht_la_SOURCES = \ listener.h \ request.h \ search.h \ + net.h \ parsed_message.h \ node_cache.cpp \ callbacks.cpp \ @@ -37,7 +38,6 @@ nobase_include_HEADERS = \ ../include/opendht/node_cache.h \ ../include/opendht/routing_table.h \ ../include/opendht/network_engine.h \ - ../include/opendht/net.h \ ../include/opendht/scheduler.h \ ../include/opendht/rate_limiter.h \ ../include/opendht/utils.h \ diff --git a/src/net.h b/src/net.h new file mode 100644 index 00000000..b47eeb25 --- /dev/null +++ b/src/net.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2017 Savoir-faire Linux Inc. + * Author(s) : Adrien Béraud + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +namespace dht { +namespace net { + +enum class MessageType { + Error = 0, + Reply, + Ping, + FindNode, + GetValues, + AnnounceValue, + Refresh, + Listen, + ValueData +}; + +} /* namespace net */ +} /* dht */ diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 26836f14..d48e4d64 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -53,14 +53,6 @@ constexpr std::chrono::seconds NetworkEngine::RX_MAX_PACKET_TIME; constexpr std::chrono::seconds NetworkEngine::RX_TIMEOUT; const std::string NetworkEngine::my_v {"RNG1"}; -const constexpr uint16_t TransId::INVALID; - -const TransPrefix TransPrefix::PING = {"pn"}; -const TransPrefix TransPrefix::FIND_NODE = {"fn"}; -const TransPrefix TransPrefix::GET_VALUES = {"gt"}; -const TransPrefix TransPrefix::ANNOUNCE_VALUES = {"pt"}; -const TransPrefix TransPrefix::REFRESH = {"rf"}; -const TransPrefix TransPrefix::LISTEN = {"lt"}; constexpr size_t NetworkEngine::MAX_REQUESTS_PER_SEC; static const uint8_t v4prefix[16] = { @@ -95,6 +87,27 @@ set_nonblocking(int fd, int nonblocking) #endif + +/* Transaction-ids are 4-bytes long, with the first two bytes identifying + * the kind of request, and the remaining two a sequence number in + * host order. + */ +struct TransId final : public std::array { + static const constexpr uint16_t INVALID {0}; + + TransId() { std::fill(begin(), end(), 0); } + TransId(const std::array& o) { std::copy(o.begin(), o.end(), begin()); } + TransId(uint32_t id) { + *reinterpret_cast(data()) = htonl(id); + } + + uint32_t toInt() const { + return ntohl(*reinterpret_cast(&(*this)[0])); + } +}; + + + struct NetworkEngine::PartialMessage { SockAddr from; time_point start; @@ -146,14 +159,14 @@ NetworkEngine::~NetworkEngine() { } void -NetworkEngine::tellListener(Sp node, uint32_t socket_id, const InfoHash& hash, want_t want, +NetworkEngine::tellListener(Sp node, Tid socket_id, const InfoHash& hash, want_t want, const Blob& ntoken, std::vector>&& nodes, std::vector>&& nodes6, std::vector>&& values, const Query& query) { auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6); try { - sendNodesValues(node->getAddr(), TransId(socket_id), nnodes.first, nnodes.second, values, query, ntoken); + sendNodesValues(node->getAddr(), socket_id, nnodes.first, nnodes.second, values, query, ntoken); } catch (const std::overflow_error& e) { DHT_LOG.e("Can't send value: buffer not large enough !"); } @@ -401,7 +414,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro auto node = cache.getNode(msg->id, from, now, true, msg->is_client); if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { - auto rsocket = node->getSocket(msg->tid.toInt()); + auto rsocket = node->getSocket(msg->tid); auto req = node->getRequest(msg->tid); /* either response for a request or data for an opened socket */ @@ -432,10 +445,10 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro switch (msg->type) { case MessageType::Error: { - if (msg->id != zeroes and ( - (msg->error_code == DhtProtocolException::NOT_FOUND and msg->tid.matches(TransPrefix::REFRESH)) or - (msg->error_code == DhtProtocolException::UNAUTHORIZED and (msg->tid.matches(TransPrefix::ANNOUNCE_VALUES) - or msg->tid.matches(TransPrefix::LISTEN))))) + if (msg->id != zeroes and req and ( + (msg->error_code == DhtProtocolException::NOT_FOUND and req->getType() == MessageType::Refresh) or + (msg->error_code == DhtProtocolException::UNAUTHORIZED and (req->getType() == MessageType::AnnounceValue + or req->getType() == MessageType::Listen)))) { req->last_try = time_point::min(); req->reply_time = time_point::min(); @@ -513,7 +526,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro case MessageType::Listen: { DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str()); ++in_stats.listen; - RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id.toInt(), std::move(msg->query)); + RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id, std::move(msg->query)); sendListenConfirmation(from, msg->tid); break; } @@ -573,7 +586,7 @@ NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr Sp NetworkEngine::sendPing(Sp node, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::PING, node->getNewTid()}; + TransId tid (node->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -590,7 +603,7 @@ NetworkEngine::sendPing(Sp node, RequestCb&& on_done, RequestExpiredCb&& o pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, node, + auto req = std::make_shared(MessageType::Ping, tid.toInt(), node, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&&) { DHT_LOG.d(req_status.node->id, "[node %s] got pong !", req_status.node->toString().c_str()); @@ -610,7 +623,7 @@ NetworkEngine::sendPing(Sp node, RequestCb&& on_done, RequestExpiredCb&& o } void -NetworkEngine::sendPong(const SockAddr& addr, TransId tid) { +NetworkEngine::sendPong(const SockAddr& addr, Tid tid) { msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(4+(network?1:0)); @@ -619,8 +632,9 @@ NetworkEngine::sendPong(const SockAddr& addr, TransId tid) { pk.pack(std::string("id")); pk.pack(myid); insertAddr(pk, addr); - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t (tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("r")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -633,7 +647,7 @@ NetworkEngine::sendPong(const SockAddr& addr, TransId tid) { Sp NetworkEngine::sendFindNode(Sp n, const InfoHash& target, want_t want, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::FIND_NODE, n->getNewTid()}; + TransId tid (n->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -657,7 +671,7 @@ NetworkEngine::sendFindNode(Sp n, const InfoHash& target, want_t want, pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::FindNode, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (on_done) { @@ -679,7 +693,7 @@ NetworkEngine::sendFindNode(Sp n, const InfoHash& target, want_t want, Sp NetworkEngine::sendGetValues(Sp n, const InfoHash& info_hash, const Query& query, want_t want, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::GET_VALUES, n->getNewTid()}; + TransId tid (n->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -706,7 +720,7 @@ NetworkEngine::sendGetValues(Sp n, const InfoHash& info_hash, const Query& pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::GetValues, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (on_done) { @@ -807,7 +821,7 @@ NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, const std::vector& svals, const SockAddr& addr) +NetworkEngine::sendValueParts(const TransId& tid, const std::vector& svals, const SockAddr& addr) { msgpack::sbuffer buffer; unsigned i=0; @@ -837,7 +851,7 @@ NetworkEngine::sendValueParts(TransId tid, const std::vector& svals, const } void -NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& nodes, const Blob& nodes6, +NetworkEngine::sendNodesValues(const SockAddr& addr, Tid tid, const Blob& nodes, const Blob& nodes6, const std::vector>& st, const Query& query, const Blob& token) { msgpack::sbuffer buffer; @@ -878,8 +892,9 @@ NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& no } } - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t (tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("r")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -954,8 +969,8 @@ NetworkEngine::sendListen(Sp n, RequestExpiredCb&& on_expired, SocketCb&& socket_cb) { - uint32_t socket; - auto tid = TransId { TransPrefix::LISTEN, n->getNewTid() }; + Tid socket; + TransId tid (n->getNewTid()); if (previous and previous->node == n) { socket = previous->getSocket(); } else { @@ -994,7 +1009,7 @@ NetworkEngine::sendListen(Sp n, pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::Listen, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (on_done) @@ -1012,7 +1027,7 @@ NetworkEngine::sendListen(Sp n, } void -NetworkEngine::sendListenConfirmation(const SockAddr& addr, TransId tid) { +NetworkEngine::sendListenConfirmation(const SockAddr& addr, Tid tid) { msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(4+(network?1:0)); @@ -1021,8 +1036,9 @@ NetworkEngine::sendListenConfirmation(const SockAddr& addr, TransId tid) { pk.pack(std::string("id")); pk.pack(myid); insertAddr(pk, addr); - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t (tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("r")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -1041,7 +1057,7 @@ NetworkEngine::sendAnnounceValue(Sp n, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::ANNOUNCE_VALUES, n->getNewTid()}; + TransId tid (n->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -1065,7 +1081,7 @@ NetworkEngine::sendAnnounceValue(Sp n, pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::AnnounceValue, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (msg.value_id == Value::INVALID_ID) { @@ -1099,7 +1115,7 @@ NetworkEngine::sendRefreshValue(Sp n, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::REFRESH, n->getNewTid()}; + TransId tid (n->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -1119,7 +1135,7 @@ NetworkEngine::sendRefreshValue(Sp n, pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::Refresh, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (msg.value_id == Value::INVALID_ID) { @@ -1144,7 +1160,7 @@ NetworkEngine::sendRefreshValue(Sp n, } void -NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id vid) { +NetworkEngine::sendValueAnnounced(const SockAddr& addr, Tid tid, Value::Id vid) { msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(4+(network?1:0)); @@ -1154,8 +1170,9 @@ NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id v pk.pack(std::string("vid")); pk.pack(vid); insertAddr(pk, addr); - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t(tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("r")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -1167,7 +1184,7 @@ NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id v void NetworkEngine::sendError(const SockAddr& addr, - TransId tid, + Tid tid, uint16_t code, const std::string& message, bool include_id) @@ -1185,8 +1202,9 @@ NetworkEngine::sendError(const SockAddr& addr, pk.pack(std::string("id")); pk.pack(myid); } - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t(tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("e")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -1197,7 +1215,7 @@ NetworkEngine::sendError(const SockAddr& addr, } void -NetworkEngine::maintainRxBuffer(const TransId& tid) +NetworkEngine::maintainRxBuffer(Tid tid) { auto msg = partial_messages.find(tid); if (msg != partial_messages.end()) { diff --git a/src/node.cpp b/src/node.cpp index ba8f4cb9..8bd17a24 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -35,7 +35,7 @@ Node::Node(const InfoHash& id, const SockAddr& addr, bool client) : id(id), addr(addr), is_client(client), sockets_() { crypto::random_device rd; - transaction_id = std::uniform_int_distribution{1}(rd); + transaction_id = std::uniform_int_distribution{1}(rd); } /* This is our definition of a known-good node. */ @@ -101,7 +101,7 @@ Node::received(time_point now, const Sp& req) } Sp -Node::getRequest(const net::TransId& tid) +Node::getRequest(Tid tid) { auto it = requests_.find(tid); return it != requests_.end() ? it->second : nullptr; @@ -128,7 +128,7 @@ Node::setExpired() sockets_.clear(); } -SocketId +Tid Node::openSocket(SocketCb&& cb) { if (++transaction_id == 0) @@ -142,14 +142,14 @@ Node::openSocket(SocketCb&& cb) } Socket* -Node::getSocket(SocketId id) +Node::getSocket(Tid id) { auto it = sockets_.find(id); return it == sockets_.end() ? nullptr : &it->second; } void -Node::closeSocket(SocketId id) +Node::closeSocket(Tid id) { if (id) { sockets_.erase(id); diff --git a/src/parsed_message.h b/src/parsed_message.h index 1164342e..983ffefa 100644 --- a/src/parsed_message.h +++ b/src/parsed_message.h @@ -26,17 +26,14 @@ namespace dht { namespace net { -enum class MessageType { - Error = 0, - Reply, - Ping, - FindNode, - GetValues, - AnnounceValue, - Refresh, - Listen, - ValueData -}; +Tid unpackTid(const msgpack::object& o) { + switch (o.type) { + case msgpack::type::POSITIVE_INTEGER: + return o.as(); + default: + return ntohl(*reinterpret_cast(o.as>().data())); + } +} struct ParsedMessage { MessageType type; @@ -51,9 +48,9 @@ struct ParsedMessage { /* target id around which to find nodes */ InfoHash target; /* transaction id */ - TransId tid; + Tid tid; /* tid for packets going through request socket */ - TransId socket_id; + Tid socket_id; /* security token */ Blob token; /* the value id (announce confirmation) */ diff --git a/src/request.h b/src/request.h index 0b222eba..3c04cd34 100644 --- a/src/request.h +++ b/src/request.h @@ -59,18 +59,19 @@ struct Request { State getState() const { return state_; } Request(State state = State::PENDING) : state_(state) {} - Request(const TransId& tid, + Request(MessageType type, Tid tid, Sp node, Blob&& msg, std::function on_done, std::function on_expired, - uint32_t socket = 0) : - node(node), on_done(on_done), on_expired(on_expired), tid(tid), msg(std::move(msg)), socket(socket) { } + Tid socket = 0) : + node(node), tid(tid), type(type), on_done(on_done), on_expired(on_expired), msg(std::move(msg)), socket(socket) { } - TransId getTid() const { return tid; } + Tid getTid() const { return tid; } + MessageType getType() const { return type; } - uint32_t getSocket() { return socket; } - uint32_t closeSocket() { auto ret = socket; socket = 0; return ret; } + Tid getSocket() { return socket; } + Tid closeSocket() { auto ret = socket; socket = 0; return ret; } void setExpired() { if (pending()) { @@ -107,6 +108,8 @@ private: msg.clear(); } + const Tid tid {0}; /* the request id. */ + const MessageType type {}; State state_ {State::PENDING}; unsigned attempt_count {0}; /* number of attempt to process the request. */ @@ -116,9 +119,8 @@ private: std::function on_done {}; std::function on_expired {}; - const TransId tid; /* the request id. */ Blob msg {}; /* the serialized message. */ - uint32_t socket; /* the socket used for further reponses. */ + Tid socket; /* the socket used for further reponses. */ }; } /* namespace net */ diff --git a/src/utils.cpp b/src/utils.cpp index cb2fd5b3..87826e1f 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -19,7 +19,6 @@ #include "utils.h" #include "sockaddr.h" #include "default_types.h" -#include "net.h" /* An IPv4 equivalent to IN6_IS_ADDR_UNSPECIFIED */ #ifndef IN_IS_ADDR_UNSPECIFIED @@ -161,18 +160,4 @@ findMapValue(msgpack::object& map, const std::string& key) { return nullptr; } -namespace net { - -TransId -unpackTid(msgpack::object& o) { - switch (o.type) { - case msgpack::type::POSITIVE_INTEGER: - return o.as(); - default: - return o.as>(); - } -} - -} - } From a789ffe7ae37a638e7e551706138ae6246fbd522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 20 Oct 2017 17:46:27 +0200 Subject: [PATCH 049/156] dht: prevent use-after-free of socket callback --- src/dht.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index b79f4ecf..12fa80c9 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -553,8 +553,8 @@ Dht::searchStep(Sp sr) [this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable { /* on done */ if (auto sr = ws.lock()) { - onListenDone(req.node, answer, sr); scheduler.edit(sr->nextSearchStep, scheduler.time()); + onListenDone(req.node, answer, sr); } }, [this,ws,query](const net::Request& req, bool over) mutable @@ -569,8 +569,8 @@ Dht::searchStep(Sp sr) [this,ws,query](const Sp& node, net::RequestAnswer&& answer) mutable { /* on new values */ if (auto sr = ws.lock()) { - onGetValuesDone(node, answer, sr, query); scheduler.edit(sr->nextSearchStep, scheduler.time()); + onGetValuesDone(node, answer, sr, query); } } ); From fe09146609a727d537d79f6b3f336ee3edfda3bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Fri, 20 Oct 2017 14:51:35 -0400 Subject: [PATCH 050/156] rate_limiter: typo fix, 'Perdiod'->'Period' --- include/opendht/rate_limiter.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/opendht/rate_limiter.h b/include/opendht/rate_limiter.h index 061b36a5..17ba4bbe 100644 --- a/include/opendht/rate_limiter.h +++ b/include/opendht/rate_limiter.h @@ -23,12 +23,12 @@ namespace dht { -template +template class RateLimiter { public: /** Clear outdated records and return current quota usage */ size_t maintain(const time_point& now) { - auto limit = now - std::chrono::seconds(Perdiod); + auto limit = now - std::chrono::seconds(Period); while (not records.empty() and records.front() < limit) records.pop(); return records.size(); From f4ebbc66515ad561b70919c8a5cc5a0467480f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Sat, 21 Oct 2017 01:30:12 +0200 Subject: [PATCH 051/156] update to version 1.4.1 * updated MSVC project * bug fixes --- CMakeLists.txt | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03d21b71..2ea5b834 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required (VERSION 3.1) project (opendht) set (opendht_VERSION_MAJOR 1) -set (opendht_VERSION_MINOR 4.0) +set (opendht_VERSION_MINOR 4.1) set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR}) set (PACKAGE_VERSION ${opendht_VERSION}) set (VERSION "${opendht_VERSION}") diff --git a/configure.ac b/configure.ac index 31a628b7..504de170 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl define macros m4_define([opendht_major_version], 1) m4_define([opendht_minor_version], 4) -m4_define([opendht_patch_version], 0) +m4_define([opendht_patch_version], 1) m4_define([opendht_version], [opendht_major_version.opendht_minor_version.opendht_patch_version]) From cd17858ae13161008ab3898cb258b782a53077c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Sat, 21 Oct 2017 18:00:46 +0200 Subject: [PATCH 052/156] dht: periodic cleanup of unused quota entries --- src/dht.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index 12fa80c9..9c97bae8 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -1291,10 +1291,17 @@ Dht::expireStore() DHT_LOG.w("Discarded %ld bytes, still %ld used", largest->first.toString().c_str(), total_store_size); if (ret.values_diff) break; - } else - std::cout << "exp_value not found " << exp_value.first << std::endl; + } } } + + // remove unused quota entires + for (auto i = store_quota.begin(); i != store_quota.end();) { + if (i->second.size() == 0) + i = store_quota.erase(i); + else + ++i; + } } void From 63366722a930983e511295d4216cdddd6ebb238e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Sat, 21 Oct 2017 18:01:32 +0200 Subject: [PATCH 053/156] dht: reduce verbosity of storage log --- include/opendht/dht.h | 1 - src/dht.cpp | 27 +++++++++------------------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/include/opendht/dht.h b/include/opendht/dht.h index 9c8b38d6..d5131481 100644 --- a/include/opendht/dht.h +++ b/include/opendht/dht.h @@ -428,7 +428,6 @@ private: void storageChanged(const InfoHash& id, Storage& st, ValueStorage&); std::string printStorageLog(const decltype(store)::value_type&) const; - void printStorageQuota(std::ostream& out, const decltype(store_quota)::value_type& ip) const; /** * For a given storage, if values don't belong there anymore because this diff --git a/src/dht.cpp b/src/dht.cpp index 9c97bae8..bf5ee806 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -1256,8 +1256,7 @@ void Dht::expireStore() { // removing expired values - auto i = store.begin(); - while (i != store.end()) { + for (auto i = store.begin(); i != store.end();) { expireStore(i); if (i->second.empty() && i->second.listeners.empty() && i->second.local_listeners.empty()) { @@ -1541,12 +1540,6 @@ Dht::dumpTables() const DHT_LOG.d("%s", out.str().c_str()); } -void -Dht::printStorageQuota(std::ostream& out, const decltype(store_quota)::value_type& ip) const -{ - out << "IP " << ip.first.toString() << " uses " << ip.second.size() << " bytes" << std::endl; -} - std::string Dht::getStorageLog() const { @@ -1559,9 +1552,12 @@ Dht::getStorageLog() const else out << (total_store_size/1024) << " KB)"; out << std::endl << std::endl; - for (const auto& ip : store_quota) { - printStorageQuota(out, ip); - } + std::multimap q_map; + for (const auto& ip : store_quota) + if (ip.second.size()) + q_map.emplace(ip.second.size(), &ip.first); + for (auto ip = q_map.rbegin(); ip != q_map.rend(); ++ip) + out << "IP " << ip->second->toString() << " uses " << ip->first << " bytes" << std::endl; out << std::endl; return out.str(); } @@ -1593,12 +1589,7 @@ Dht::printStorageLog(const decltype(store)::value_type& s) const const auto& now = scheduler.time(); for (const auto& node_listeners : st.listeners) { const auto& node = node_listeners.first; - for (const auto& l : node_listeners.second) { - out << " " << "Listener " << node->toString(); - auto since = duration_cast(now - l.second.time); - auto expires = duration_cast(l.second.time + Node::NODE_EXPIRE_TIME - now); - out << " (since " << since.count() << "s, exp in " << expires.count() << "s)" << std::endl; - } + out << " " << "Listener " << node->toString() << " : " << node_listeners.second.size() << " entries" << std::endl; } return out.str(); } @@ -2214,7 +2205,7 @@ Dht::onListen(Sp node, const InfoHash& hash, const Blob& token, size_t soc net::DhtProtocolException::LISTEN_NO_INFOHASH }; } - if (!tokenMatch(token, node->getAddr())) { + if (not tokenMatch(token, node->getAddr())) { DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'listen'", node->toString().c_str(), hash.toString().c_str()); throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::LISTEN_WRONG_TOKEN}; } From 64f0aa1532a253b4e804dad91aca3f1e57e6d3a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 30 Oct 2017 13:12:12 -0400 Subject: [PATCH 054/156] parsed message: don't override the reported address family --- src/parsed_message.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/parsed_message.h b/src/parsed_message.h index 983ffefa..f30f27b2 100644 --- a/src/parsed_message.h +++ b/src/parsed_message.h @@ -229,13 +229,11 @@ ParsedMessage::msgpack_unpack(msgpack::object msg) if (l == sizeof(in_addr)) { addr.setFamily(AF_INET); auto& a = addr.getIPv4(); - std::fill_n((uint8_t*)&a, sizeof(a), 0); a.sin_port = 0; std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin_addr); } else if (l == sizeof(in6_addr)) { addr.setFamily(AF_INET6); auto& a = addr.getIPv6(); - std::fill_n((uint8_t*)&a, sizeof(a), 0); a.sin6_port = 0; std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin6_addr); } From 6378840d330a70676f7ae97b68e05efa7ca0f87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 25 Oct 2017 13:12:21 +0200 Subject: [PATCH 055/156] dhtnode: allow to cancel a listen operation --- src/dht.cpp | 3 +-- tools/dhtnode.cpp | 12 ++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index bf5ee806..2054f7c8 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -1586,7 +1586,6 @@ Dht::printStorageLog(const decltype(store)::value_type& s) const << st.totalSize() << " bytes)" << std::endl; if (not st.local_listeners.empty()) out << " " << st.local_listeners.size() << " local listeners" << std::endl; - const auto& now = scheduler.time(); for (const auto& node_listeners : st.listeners) { const auto& node = node_listeners.first; out << " " << "Listener " << node->toString() << " : " << node_listeners.second.size() << " entries" << std::endl; @@ -1966,7 +1965,7 @@ Dht::importValues(const std::vector& import) throw msgpack::type_error(); for (unsigned i = 0; i < valarr.via.array.size; i++) { auto& valel = valarr.via.array.ptr[i]; - if (valel.via.array.size < 2) + if (valel.type != msgpack::type::ARRAY or valel.via.array.size < 2) throw msgpack::type_error(); time_point val_time; Value tmp_val; diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 7e42c641..8f41eaef 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -165,7 +165,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) if (op.empty()) continue; - static const std::set VALID_OPS {"g", "l", "il", "ii", "p", "s", "e", "a"}; + static const std::set VALID_OPS {"g", "l", "cl", "il", "ii", "p", "s", "e", "a"}; if (VALID_OPS.find(op) == VALID_OPS.cend()) { std::cout << "Unknown command: " << op << std::endl; std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl; @@ -236,11 +236,19 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) else if (op == "l") { std::string rem; std::getline(iss, rem); - dht->listen(id, [](std::shared_ptr value) { + auto token = dht->listen(id, [](std::shared_ptr value) { std::cout << "Listen: found value:" << std::endl; std::cout << "\t" << *value << std::endl; return true; }, {}, dht::Where {std::move(rem)}); + std::cout << "Listening, token: " << token.get() << std::endl; + } + else if (op == "cl") { + std::string rem; + iss >> rem; + size_t token = std::stoul(rem); + std::cout << "Cancel listen, token: " << token << " " << rem << std::endl; + dht->cancelListen(id, token); } else if (op == "p") { std::string v; From 41ebbe21d4123cb44efd14cc6e588073dcc0e7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 26 Oct 2017 19:24:28 +0200 Subject: [PATCH 056/156] node cache: expire nodes on destructions Expiring nodes on cache destruction allow to cancel any remaining request before the dht instance is destructed, since Node instances might remain alive after the dht instance is destroyed. --- include/opendht/node_cache.h | 3 +++ src/node_cache.cpp | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/include/opendht/node_cache.h b/include/opendht/node_cache.h index 97e62cd1..9b1e2a06 100644 --- a/include/opendht/node_cache.h +++ b/include/opendht/node_cache.h @@ -37,12 +37,15 @@ struct NodeCache { */ void clearBadNodes(sa_family_t family = 0); + ~NodeCache(); + private: class NodeMap : public std::map> { public: Sp getNode(const InfoHash& id); Sp getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client); void clearBadNodes(); + void setExpired(); }; const NodeMap& cache(sa_family_t af) const { return af == AF_INET ? cache_4 : cache_6; } diff --git a/src/node_cache.cpp b/src/node_cache.cpp index 68b6441b..485b208b 100644 --- a/src/node_cache.cpp +++ b/src/node_cache.cpp @@ -20,6 +20,12 @@ namespace dht { +NodeCache::~NodeCache() +{ + cache_4.setExpired(); + cache_6.setExpired(); +} + Sp NodeCache::getNode(const InfoHash& id, sa_family_t family) { return cache(family).getNode(id); @@ -116,4 +122,12 @@ NodeCache::NodeMap::clearBadNodes() { } } +void +NodeCache::NodeMap::setExpired() { + for (auto& wn : *this) + if (auto n = wn.second.lock()) + n->setExpired(); + clear(); +} + } From a401aea132aa963de4a7684714c3a5d35f2212ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 26 Oct 2017 17:12:19 +0200 Subject: [PATCH 057/156] dht: simplify searchSendAnnounceValue --- src/dht.cpp | 120 ++++++++++++++++++++++++++-------------------------- 1 file changed, 61 insertions(+), 59 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index 2054f7c8..85790652 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -386,11 +386,12 @@ void Dht::searchSendAnnounceValue(const Sp& sr) { std::weak_ptr ws = sr; const auto& now = scheduler.time(); for (auto& n : sr->nodes) { - auto something_to_announce = std::find_if(sr->announce.cbegin(), sr->announce.cend(), - [this,&now,&sr,&n](const Announce& a) { - return n.isSynced(now) and n.getAnnounceTime(a.value->id) <= now; - }) != sr->announce.cend(); - if (not something_to_announce) + if (not n.isSynced(now)) + continue; + if (std::find_if(sr->announce.cbegin(), sr->announce.cend(), + [this,&now,&n](const Announce& a) { + return n.getAnnounceTime(a.value->id) <= now; + }) == sr->announce.cend()) continue; auto onDone = [this,ws](const net::Request& req, net::RequestAnswer&& answer) @@ -409,62 +410,63 @@ void Dht::searchSendAnnounceValue(const Sp& sr) { auto onSelectDone = [this,ws,onDone,onExpired](const net::Request& req, net::RequestAnswer&& answer) mutable { /* on probing done */ + auto sr = ws.lock(); + if (not sr) return; + auto sn = sr->getNode(req.node); + if (not sn) return; + const auto& now = scheduler.time(); - if (auto sr = ws.lock()) { - if (auto sn = sr->getNode(req.node)) { - for (auto ait = sr->announce.begin(); ait != sr->announce.end();) { - auto& a = *ait; - if (sn->isSynced(now) and sn->getAnnounceTime(a.value->id) <= now) { - bool hasValue {false}; - uint16_t seq_no = 0; - try { - const auto& f = std::find_if(answer.fields.cbegin(), answer.fields.cend(), - [&a](const Sp& i){ - return i->index.at(Value::Field::Id).getInt() == a.value->id; - }); - if (f != answer.fields.cend() and *f) { - hasValue = true; - seq_no = static_cast((*f)->index.at(Value::Field::SeqNum).getInt()); - } - } catch (std::out_of_range&) { } - - auto next_refresh_time = now + getType(a.value->type).expiration; - /* only put the value if the node doesn't already have it */ - if (not hasValue or seq_no < a.value->seq) { - DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'put' (vid: %d)", - sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); - sn->acked[a.value->id] = std::make_pair(network_engine.sendAnnounceValue(sn->node, - sr->id, - a.value, - a.permanent ? time_point::max() : a.created, - sn->token, - onDone, - onExpired), next_refresh_time); - } else if (hasValue and a.permanent) { - DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'refresh' (vid: %d)", - sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); - sn->acked[a.value->id] = std::make_pair(network_engine.sendRefreshValue(sn->node, - sr->id, - a.value->id, - sn->token, - onDone, - onExpired), next_refresh_time); - } else { - DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] already has value (vid: %d). Aborting.", - sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); - auto ack_req = std::make_shared(net::Request::State::COMPLETED); - ack_req->reply_time = now; - sn->acked[a.value->id] = std::make_pair(std::move(ack_req), next_refresh_time); - - /* step to clear announces */ - scheduler.edit(sr->nextSearchStep, now); - } - } else { - /* Search is now unsynced. Let's call searchStep to sync again. */ - scheduler.edit(sr->nextSearchStep, sr->getNextStepTime(now)); - } - ++ait; + if (not sn->isSynced(now)) { + /* Search is now unsynced. Let's call searchStep to sync again. */ + scheduler.edit(sr->nextSearchStep, sr->getNextStepTime(now)); + return; + } + for (auto& a : sr->announce) { + if (sn->getAnnounceTime(a.value->id) > now) + continue; + bool hasValue {false}; + uint16_t seq_no = 0; + try { + const auto& f = std::find_if(answer.fields.cbegin(), answer.fields.cend(), + [&a](const Sp& i){ + return i->index.at(Value::Field::Id).getInt() == a.value->id; + }); + if (f != answer.fields.cend() and *f) { + hasValue = true; + seq_no = static_cast((*f)->index.at(Value::Field::SeqNum).getInt()); } + } catch (std::out_of_range&) { } + + auto next_refresh_time = now + getType(a.value->type).expiration; + /* only put the value if the node doesn't already have it */ + if (not hasValue or seq_no < a.value->seq) { + DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'put' (vid: %d)", + sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); + sn->acked[a.value->id] = std::make_pair(network_engine.sendAnnounceValue(sn->node, + sr->id, + a.value, + a.permanent ? time_point::max() : a.created, + sn->token, + onDone, + onExpired), next_refresh_time); + } else if (hasValue and a.permanent) { + DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'refresh' (vid: %d)", + sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); + sn->acked[a.value->id] = std::make_pair(network_engine.sendRefreshValue(sn->node, + sr->id, + a.value->id, + sn->token, + onDone, + onExpired), next_refresh_time); + } else { + DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] already has value (vid: %d). Aborting.", + sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); + auto ack_req = std::make_shared(net::Request::State::COMPLETED); + ack_req->reply_time = now; + sn->acked[a.value->id] = std::make_pair(std::move(ack_req), next_refresh_time); + + /* step to clear announces */ + scheduler.edit(sr->nextSearchStep, now); } } }; From eb9f314356c9200270f04a3737c78e0fb02a6081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 26 Oct 2017 18:48:23 +0200 Subject: [PATCH 058/156] dhtnode: allow to perform and cancel persistent put --- tools/dhtnode.cpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 8f41eaef..96cb394d 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -62,6 +62,7 @@ void print_help() { << " g Get values at ." << std::endl << " l Listen for value changes at ." << std::endl << " p Put string value at ." << std::endl + << " pp Put string value at (persistent version)." << std::endl << " s Put string value at , signed with our generated private key." << std::endl << " e Put string value at , encrypted for with its public key (if found)." << std::endl; std::cout << std::endl << "Indexation operations on the DHT:" << std::endl @@ -165,7 +166,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) if (op.empty()) continue; - static const std::set VALID_OPS {"g", "l", "cl", "il", "ii", "p", "s", "e", "a"}; + static const std::set VALID_OPS {"g", "l", "cl", "il", "ii", "p", "pp", "cpp", "s", "e", "a"}; if (VALID_OPS.find(op) == VALID_OPS.cend()) { std::cout << "Unknown command: " << op << std::endl; std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl; @@ -246,9 +247,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) else if (op == "cl") { std::string rem; iss >> rem; - size_t token = std::stoul(rem); - std::cout << "Cancel listen, token: " << token << " " << rem << std::endl; - dht->cancelListen(id, token); + dht->cancelListen(id, std::stoul(rem)); } else if (op == "p") { std::string v; @@ -261,6 +260,25 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) std::cout << "Put: " << (ok ? "success" : "failure") << " (took " << print_dt(end-start) << "s)" << std::endl; }); } + else if (op == "pp") { + std::string v; + iss >> v; + auto value = std::make_shared( + dht::ValueType::USER_DATA.id, + std::vector {v.begin(), v.end()} + ); + dht->put(id, value, [start,value](bool ok) { + auto end = std::chrono::high_resolution_clock::now(); + auto flags(std::cout.flags()); + std::cout << "Put: " << (ok ? "success" : "failure") << " (took " << print_dt(end-start) << "s). Value ID: " << std::hex << value->id << std::endl; + std::cout.flags(flags); + }, time_point::max(), true); + } + else if (op == "cpp") { + std::string rem; + iss >> rem; + dht->cancelPut(id, std::stoul(rem, nullptr, 16)); + } else if (op == "s") { if (not params.generate_identity) { print_id_req(); From 0a130c70458db29b3cbb3e2fd4a3f4fedee6b8c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 26 Oct 2017 18:49:46 +0200 Subject: [PATCH 059/156] dht: update permanent state on announce --- src/dht.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dht.cpp b/src/dht.cpp index 85790652..46f8326b 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -743,8 +743,9 @@ Dht::announce(const InfoHash& id, n.probe_query.reset(); n.acked[value->id].first.reset(); } - } - else { + } else { + a_sr->permanent = permanent; + a_sr->created = created; if (a_sr->value != value) { a_sr->value = value; for (auto& n : sr->nodes) { @@ -756,9 +757,8 @@ Dht::announce(const InfoHash& id, if (a_sr->callback) a_sr->callback(true, {}); a_sr->callback = {}; - if (callback) { + if (callback) callback(true, {}); - } return; } else { if (a_sr->callback) From 8cd3ff9fc47b8797d369b405a8f294fa61355093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 30 Oct 2017 10:46:24 -0400 Subject: [PATCH 060/156] network engine: remove unused constant --- src/network_engine.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/network_engine.cpp b/src/network_engine.cpp index d48e4d64..fa198730 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -93,8 +93,6 @@ set_nonblocking(int fd, int nonblocking) * host order. */ struct TransId final : public std::array { - static const constexpr uint16_t INVALID {0}; - TransId() { std::fill(begin(), end(), 0); } TransId(const std::array& o) { std::copy(o.begin(), o.end(), begin()); } TransId(uint32_t id) { @@ -107,7 +105,6 @@ struct TransId final : public std::array { }; - struct NetworkEngine::PartialMessage { SockAddr from; time_point start; From fcaaa01b2d40011df650cd092cb841e825c4e49c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 30 Oct 2017 12:47:46 -0400 Subject: [PATCH 061/156] network engine: add "value update" message type --- src/net.h | 3 ++- src/network_engine.cpp | 11 ++++++++++- src/parsed_message.h | 7 +++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/net.h b/src/net.h index b47eeb25..0aa9dda9 100644 --- a/src/net.h +++ b/src/net.h @@ -30,7 +30,8 @@ enum class MessageType { AnnounceValue, Refresh, Listen, - ValueData + ValueData, + ValueUpdate }; } /* namespace net */ diff --git a/src/network_engine.cpp b/src/network_engine.cpp index fa198730..cab07ec1 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -410,7 +410,16 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro const auto& now = scheduler.time(); auto node = cache.getNode(msg->id, from, now, true, msg->is_client); - if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { + if (msg->type == MessageType::ValueUpdate) { + auto rsocket = node->getSocket(msg->tid); + if (not rsocket) + throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find socket", msg->id}; + node->received(now, {}); + onNewNode(node, 2); + deserializeNodes(*msg, from); + rsocket->on_receive(node, std::move(*msg)); + } + else if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { auto rsocket = node->getSocket(msg->tid); auto req = node->getRequest(msg->tid); diff --git a/src/parsed_message.h b/src/parsed_message.h index f30f27b2..c4ac56cf 100644 --- a/src/parsed_message.h +++ b/src/parsed_message.h @@ -125,6 +125,7 @@ ParsedMessage::msgpack_unpack(msgpack::object msg) { auto y = findMapValue(msg, "y"); auto r = findMapValue(msg, "r"); + auto u = findMapValue(msg, "u"); auto e = findMapValue(msg, "e"); auto v = findMapValue(msg, "p"); @@ -153,6 +154,8 @@ ParsedMessage::msgpack_unpack(msgpack::object msg) type = MessageType::Reply; else if (v) type = MessageType::ValueData; + else if (u) + type = MessageType::ValueUpdate; else if (y and y->as() != "q") throw msgpack::type_error(); else if (q == "ping") @@ -185,9 +188,9 @@ ParsedMessage::msgpack_unpack(msgpack::object msg) } auto a = findMapValue(msg, "a"); - if (!a && !r && !e) + if (!a && !r && !e && !u) throw msgpack::type_error(); - auto& req = a ? *a : (r ? *r : *e); + auto& req = a ? *a : (r ? *r : (u ? *u : *e)); if (e) { if (e->type != msgpack::type::ARRAY) From dc99919338d697da6b1ec0fe1d53292f742a4c87 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Wed, 1 Nov 2017 11:59:11 -0400 Subject: [PATCH 062/156] InfoHash: add explicit operator bool() --- include/opendht/default_types.h | 2 +- include/opendht/infohash.h | 12 +++++++++++- include/opendht/value.h | 4 ++-- src/dht.cpp | 8 ++++---- src/network_engine.cpp | 12 ++++++------ src/node_cache.cpp | 2 +- src/securedht.cpp | 2 +- src/value.cpp | 2 +- 8 files changed, 27 insertions(+), 17 deletions(-) diff --git a/include/opendht/default_types.h b/include/opendht/default_types.h index bef2f659..58595163 100644 --- a/include/opendht/default_types.h +++ b/include/opendht/default_types.h @@ -93,7 +93,7 @@ public: static Value::Filter getFilter() { return Value::Filter::chain( BaseClass::getFilter(), - [](const Value& v){ return v.recipient != InfoHash(); } + [](const Value& v) { return static_cast(v.recipient); } ); } diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index ab50d5a7..57671186 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -138,7 +138,7 @@ using PkId = h256; */ class OPENDHT_PUBLIC InfoHash final : public std::array { public: - constexpr InfoHash() : std::array() {} + constexpr InfoHash() : std::array({{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}}) {} constexpr InfoHash(const std::array& h) : std::array(h) {} InfoHash(const uint8_t* h, size_t h_len) : std::array() { if (h_len < HASH_LEN) @@ -267,6 +267,16 @@ public: return false; } + explicit operator bool() const { + auto a = reinterpret_cast(data()); + auto b = reinterpret_cast(data() + size()); + for (; a != b; a++) { + if (*a) + return true; + } + return false; + } + static inline InfoHash get(const std::string& data) { return get((const uint8_t*)data.data(), data.size()); } diff --git a/include/opendht/value.h b/include/opendht/value.h index 9fd10dfa..6e5a33db 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -424,11 +424,11 @@ struct OPENDHT_PUBLIC Value void msgpack_pack_to_sign(Packer& pk) const { bool has_owner = owner && *owner; - pk.pack_map((user_type.empty()?0:1) + (has_owner?(recipient == InfoHash() ? 4 : 5):2)); + pk.pack_map((user_type.empty()?0:1) + (has_owner?(recipient ? 5 : 4):2)); if (has_owner) { // isSigned pk.pack(std::string("seq")); pk.pack(seq); pk.pack(std::string("owner")); owner->msgpack_pack(pk); - if (recipient != InfoHash()) { + if (recipient) { pk.pack(std::string("to")); pk.pack(recipient); } } diff --git a/src/dht.cpp b/src/dht.cpp index 46f8326b..6dec3b26 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -1655,7 +1655,7 @@ Dht::~Dht() Dht::Dht() : store(), scheduler(DHT_LOG), network_engine(DHT_LOG, scheduler) {} Dht::Dht(int s, int s6, Config config) - : myid(config.node_id != zeroes ? config.node_id : InfoHash::getRandom()), + : myid(config.node_id ? config.node_id : InfoHash::getRandom()), is_bootstrap(config.is_bootstrap), maintain_storage(config.maintain_storage), store(), store_quota(), scheduler(DHT_LOG), @@ -2102,7 +2102,7 @@ Dht::onFindNode(Sp node, const InfoHash& target, want_t want) net::RequestAnswer Dht::onGetValues(Sp node, const InfoHash& hash, want_t, const Query& query) { - if (hash == zeroes) { + if (not hash) { DHT_LOG.w("[node %s] Eek! Got get_values with no info_hash", node->toString().c_str()); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, @@ -2199,7 +2199,7 @@ void Dht::onGetValuesDone(const Sp& node, net::RequestAnswer Dht::onListen(Sp node, const InfoHash& hash, const Blob& token, size_t socket_id, const Query& query) { - if (hash == zeroes) { + if (not hash) { DHT_LOG.w(node->id, "[node %s] listen with no info_hash", node->toString().c_str()); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, @@ -2238,7 +2238,7 @@ Dht::onAnnounce(Sp n, const time_point& creation_date) { auto& node = *n; - if (hash == zeroes) { + if (not hash) { DHT_LOG.w(node.id, "put with no info_hash"); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, diff --git a/src/network_engine.cpp b/src/network_engine.cpp index cab07ec1..93a6cf88 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -206,7 +206,7 @@ NetworkEngine::requestStep(Sp sreq) if (req.isExpired(now)) { DHT_LOG.e(node.id, "[node %s] expired !", node.toString().c_str()); node.setExpired(); - if (node.id == zeroes) + if (not node.id) requests.erase(req.tid); return; } else if (req.attempt_count == 1) { @@ -221,7 +221,7 @@ NetworkEngine::requestStep(Sp sreq) err == EAFNOSUPPORT) { node.setExpired(); - if (node.id == zeroes) + if (not node.id) requests.erase(req.tid); } else { if (err != EAGAIN) { @@ -244,7 +244,7 @@ void NetworkEngine::sendRequest(const Sp& request) { auto& node = request->node; - if (node->id == zeroes) + if (not node->id) requests.emplace(request->tid, request); request->start = scheduler.time(); node->requested(request); @@ -373,7 +373,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr& return; } - if (msg->id == myid || msg->id == zeroes) { + if (msg->id == myid or not msg->id) { DHT_LOG.d("Received message from self"); return; } @@ -426,7 +426,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro /* either response for a request or data for an opened socket */ if (not req and not rsocket) { auto req_it = requests.find(msg->tid); - if (req_it != requests.end() and req_it->second->node->id == zeroes) { + if (req_it != requests.end() and not req_it->second->node->id) { req = req_it->second; req->node = node; requests.erase(req_it); @@ -451,7 +451,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro switch (msg->type) { case MessageType::Error: { - if (msg->id != zeroes and req and ( + if (msg->id and req and ( (msg->error_code == DhtProtocolException::NOT_FOUND and req->getType() == MessageType::Refresh) or (msg->error_code == DhtProtocolException::UNAUTHORIZED and (req->getType() == MessageType::AnnounceValue or req->getType() == MessageType::Listen)))) diff --git a/src/node_cache.cpp b/src/node_cache.cpp index 485b208b..926768da 100644 --- a/src/node_cache.cpp +++ b/src/node_cache.cpp @@ -33,7 +33,7 @@ NodeCache::getNode(const InfoHash& id, sa_family_t family) { Sp NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) { - if (id == zeroes) + if (not id) return std::make_shared(id, addr); return cache(addr.getFamily()).getNode(id, addr, now, confirm, client); } diff --git a/src/securedht.cpp b/src/securedht.cpp index ea15e125..88a1b118 100644 --- a/src/securedht.cpp +++ b/src/securedht.cpp @@ -35,7 +35,7 @@ namespace dht { Config& getConfig(SecureDht::Config& conf) { auto& c = conf.node_config; - if (c.node_id == InfoHash() and conf.id.second) + if (not c.node_id and conf.id.second) c.node_id = InfoHash::get("node:"+conf.id.second->getId().toString()); return c; } diff --git a/src/value.cpp b/src/value.cpp index 90d1aaf3..e3d3aece 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -41,7 +41,7 @@ std::ostream& operator<< (std::ostream& s, const Value& v) s << "encrypted "; else if (v.isSigned()) { s << "signed (v" << v.seq << ") "; - if (v.recipient != InfoHash()) + if (v.recipient) s << "decrypted "; } if (not v.isEncrypted()) { From e33ac02351e6bfdb8624d73d4f2159ed9aa2cc03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Tue, 24 Oct 2017 16:29:24 -0400 Subject: [PATCH 063/156] test restbed --- .gitignore | 5 ++ CMakeLists.txt | 8 ++- include/opendht/dhtproxyserver.h | 37 +++++++++++++ include/opendht/dhtrunner.h | 5 ++ src/dhtproxyserver.cpp | 90 ++++++++++++++++++++++++++++++++ src/dhtrunner.cpp | 7 +++ tools/dhtchat.cpp | 2 + 7 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 include/opendht/dhtproxyserver.h create mode 100644 src/dhtproxyserver.cpp diff --git a/.gitignore b/.gitignore index 9d50e4d5..c79e0d15 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,11 @@ Makefile.in # Python *.pyc +python/opendht.cpp +python/setup.py + +# Doxygen +doc/Doxyfile # git backup files *.orig diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ea5b834..373a4c3d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,6 +104,7 @@ list (APPEND opendht_SOURCES src/node.cpp src/value.cpp src/dht.cpp + src/dhtproxyserver.cpp src/storage.h src/listener.h src/search.h @@ -131,6 +132,7 @@ list (APPEND opendht_HEADERS include/opendht/node.h include/opendht/value.h include/opendht/dht.h + include/opendht/dhtproxyserver.h include/opendht/callbacks.h include/opendht/routing_table.h include/opendht/node_cache.h @@ -177,7 +179,8 @@ if (OPENDHT_STATIC) target_link_libraries(opendht-static ${argon2_LIBRARIES}) target_include_directories(opendht-static SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) + # TODO clean RESTBED link + target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} -lrestbed) install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () @@ -198,7 +201,8 @@ if (OPENDHT_SHARED) target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES}) target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) + # TODO clean RESTBED link + target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} -lrestbed) install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h new file mode 100644 index 00000000..ed5699e5 --- /dev/null +++ b/include/opendht/dhtproxyserver.h @@ -0,0 +1,37 @@ +// TODO gpl + +#pragma once + +#include "def.h" + +#include +#include + +// TODO add ifdef RESTBED +#include + +namespace dht { + +class DhtRunner; + +class OPENDHT_PUBLIC DhtProxyServer +{ +public: + DhtProxyServer(DhtRunner* dht); + virtual ~DhtProxyServer(); + + DhtProxyServer(const DhtProxyServer& other) = default; + DhtProxyServer(DhtProxyServer&& other) = default; + DhtProxyServer& operator=(const DhtProxyServer& other) = default; + DhtProxyServer& operator=(DhtProxyServer&& other) = default; + +private: + void getId(const std::shared_ptr& session) const; + void getNodeId(const std::shared_ptr& session) const; + + std::thread server_thread {}; + std::unique_ptr service_; + DhtRunner* dht_; +}; + +} diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index fd3346c0..35f14423 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -36,6 +36,8 @@ #include #include +#include "dhtproxyserver.h" + namespace dht { struct Node; @@ -362,6 +364,8 @@ public: */ void join(); + void startProxyInterface(); // TODO arguments + private: static constexpr std::chrono::seconds BOOTSTRAP_PERIOD {10}; @@ -391,6 +395,7 @@ private: std::mutex sock_mtx {}; std::vector> rcv {}; + std::unique_ptr proxy_server_; /** true if currently actively boostraping */ std::atomic_bool bootstraping {false}; /* bootstrap nodes given as (host, service) pairs */ diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp new file mode 100644 index 00000000..ae771bf3 --- /dev/null +++ b/src/dhtproxyserver.cpp @@ -0,0 +1,90 @@ +// TODO GPL + +#include "dhtproxyserver.h" + +#include + +#include "dhtrunner.h" + +namespace dht { + +DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) +{ + // NOTE in c++14, use make_unique + service_ = std::unique_ptr(new restbed::Service()); + + auto resource = std::make_shared(); + resource->set_path("/getId"); + resource->set_method_handler("GET", + [this](const std::shared_ptr session) + { + this->getId(session); + } + ); + service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/getNodeId"); + resource->set_method_handler("GET", + [this](const std::shared_ptr session) + { + this->getNodeId(session); + } + ); + service_->publish(resource); + + server_thread = std::thread([this]() { + auto settings = std::make_shared(); + settings->set_port(1984); // TODO add argument and move this + settings->set_default_header("Connection", "close"); + service_->start(settings); + }); +} + +DhtProxyServer::~DhtProxyServer() +{ + if (server_thread.joinable()) + server_thread.join(); +} + +void +DhtProxyServer::getId(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + + int content_length = std::stoi(request->get_header("Content-Length", "0")); + + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + auto id = dht_->getId().toString(); + s->close(restbed::OK, id, {{"Content-Length", std::to_string(id.length())}}); + } else { + s->close(restbed::OK, "", {{"Content-Length", "0"}}); + } + } + ); +} + +void +DhtProxyServer::getNodeId(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + + int content_length = std::stoi(request->get_header("Content-Length", "0")); + + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + auto id = dht_->getNodeId().toString(); + s->close(restbed::OK, id, {{"Content-Length", std::to_string(id.length())}}); + } else { + s->close(restbed::OK, "", {{"Content-Length", "0"}}); + } + } + ); +} + + +} diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index 66b4ebbd..16883539 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -800,4 +800,11 @@ DhtRunner::findCertificate(InfoHash hash, std::function(new DhtProxyServer(this)); +} + } diff --git a/tools/dhtchat.cpp b/tools/dhtchat.cpp index 8e48723f..b64b4c95 100644 --- a/tools/dhtchat.cpp +++ b/tools/dhtchat.cpp @@ -67,6 +67,8 @@ main(int argc, char **argv) if (not params.bootstrap.first.empty()) dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str()); + dht.startProxyInterface(); + print_node_info(dht, params); std::cout << " type 'c {hash}' to join a channel" << std::endl << std::endl; From 32153b11ab5f4a3e89fedb9b4b13f1d7dee76a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 25 Oct 2017 17:32:40 -0400 Subject: [PATCH 064/156] proxy putSigned --- include/opendht/dhtproxyserver.h | 1 + src/dhtproxyserver.cpp | 74 ++++++++++++++++++++++++++++++-- tools/dhtchat.cpp | 14 +++++- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index ed5699e5..6a05b52a 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -28,6 +28,7 @@ public: private: void getId(const std::shared_ptr& session) const; void getNodeId(const std::shared_ptr& session) const; + void putSigned(const std::shared_ptr& session) const; std::thread server_thread {}; std::unique_ptr service_; diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index ae771bf3..02668092 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -4,7 +4,9 @@ #include +#include "default_types.h" #include "dhtrunner.h" +#include "msgpack.hpp" namespace dht { @@ -31,12 +33,31 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) } ); service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/putSigned"); + resource->set_method_handler("PUT", + [this](const std::shared_ptr session) + { + this->putSigned(session); + } + ); + service_->publish(resource); server_thread = std::thread([this]() { auto settings = std::make_shared(); - settings->set_port(1984); // TODO add argument and move this settings->set_default_header("Connection", "close"); - service_->start(settings); + int port = 1984; + int started = false; + while (!started) + { + try { + settings->set_port(port); // TODO add argument and move this + service_->start(settings); + started = true; + } catch (std::system_error& e) { + port += 1; + } + } }); } @@ -60,7 +81,7 @@ DhtProxyServer::getId(const std::shared_ptr& session) const auto id = dht_->getId().toString(); s->close(restbed::OK, id, {{"Content-Length", std::to_string(id.length())}}); } else { - s->close(restbed::OK, "", {{"Content-Length", "0"}}); + s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); } } ); @@ -80,7 +101,52 @@ DhtProxyServer::getNodeId(const std::shared_ptr& session) cons auto id = dht_->getNodeId().toString(); s->close(restbed::OK, id, {{"Content-Length", std::to_string(id.length())}}); } else { - s->close(restbed::OK, "", {{"Content-Length", "0"}}); + s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); + } + } + ); +} + +void +DhtProxyServer::putSigned(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + + int content_length = std::stoi(request->get_header("Content-Length", "0")); + + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + if(b.empty()) { + std::string response("Missing parameters"); + s->close(restbed::BAD_REQUEST, response, {{"Content-Length", std::to_string(response.size())}}); + } else { + restbed::Bytes buf(b); + + unsigned char delimiter = static_cast('\n'); + unsigned int idx = 0; + std::vector hashBuf; + while(idx < buf.size() && buf[idx] != delimiter) { + hashBuf.emplace_back(buf[idx]); + ++idx; + } + ++idx; + std::vector valueBuf; + while(idx < buf.size()) { + valueBuf.emplace_back(buf[idx]); + ++idx; + } + InfoHash hash(hashBuf.data(), hashBuf.size()); + msgpack::unpacked msg; + msgpack::unpack(msg, reinterpret_cast(valueBuf.data()), valueBuf.size()); + auto value = std::make_shared(msg.get()); + + dht_->putSigned(hash, value); + s->close(restbed::OK, "", {{"Content-Length", "0"}}); + } + } else { + s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); } } ); diff --git a/tools/dhtchat.cpp b/tools/dhtchat.cpp index b64b4c95..5e80bef8 100644 --- a/tools/dhtchat.cpp +++ b/tools/dhtchat.cpp @@ -20,6 +20,8 @@ #include "tools_common.h" #include +#include + extern "C" { #include } @@ -131,7 +133,17 @@ main(int argc, char **argv) std::cout << "Message publishing failed !" << std::endl; }); } else { - dht.putSigned(room, dht::ImMessage(rand_id(rd), std::move(line), now), [](bool ok) { + auto value = std::make_shared( dht::ImMessage(rand_id(rd), std::move(line), now)); + + // TODO TEMP + auto valueStr = packMsg(value); + std::cout << std::endl; + for (const auto& v: valueStr) { + std::cout << "\\x" << std::hex << static_cast(v); + } + std::cout << std::endl; + + dht.putSigned(room, value, [](bool ok) { //dht.cancelPut(room, id); if (not ok) std::cout << "Message publishing failed !" << std::endl; From 6f641073ae000e0988a32463ecc4c48a3a4f3b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Thu, 26 Oct 2017 13:52:49 -0400 Subject: [PATCH 065/156] proxy put and putEncrypted --- include/opendht/dhtproxyserver.h | 30 ++++++++ src/dhtproxyserver.cpp | 125 +++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index 6a05b52a..0021583b 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -26,9 +26,39 @@ public: DhtProxyServer& operator=(DhtProxyServer&& other) = default; private: + /** + * Method: GET + */ void getId(const std::shared_ptr& session) const; + /** + * Method: GET + */ void getNodeId(const std::shared_ptr& session) const; + /** + * Method: PUT + * Body: vector of unsigned char + * first line: InfoHash hash + * second line: Value value to sign and put + * @param session + */ void putSigned(const std::shared_ptr& session) const; + /** + * Method: PUT + * Body: vector of unsigned char + * first line: InfoHash hash + * second line: InfoHash to + * third line: Value value to encrypt and put + * @param session + */ + void putEncrypted(const std::shared_ptr& session) const; + /** + * Method: PUT + * Body: vector of unsigned char + * first line: InfoHash hash + * second line: Value value to put + * @param session + */ + void put(const std::shared_ptr& session) const; std::thread server_thread {}; std::unique_ptr service_; diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index 02668092..2a804a56 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -42,6 +42,24 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) } ); service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/putEncrypted"); + resource->set_method_handler("PUT", + [this](const std::shared_ptr session) + { + this->putEncrypted(session); + } + ); + service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/put"); + resource->set_method_handler("PUT", + [this](const std::shared_ptr session) + { + this->put(session); + } + ); + service_->publish(resource); server_thread = std::thread([this]() { auto settings = std::make_shared(); @@ -125,6 +143,7 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons restbed::Bytes buf(b); unsigned char delimiter = static_cast('\n'); + // Retrieve hash unsigned int idx = 0; std::vector hashBuf; while(idx < buf.size() && buf[idx] != delimiter) { @@ -132,11 +151,13 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons ++idx; } ++idx; + // Retrieve value std::vector valueBuf; while(idx < buf.size()) { valueBuf.emplace_back(buf[idx]); ++idx; } + // Build parameters InfoHash hash(hashBuf.data(), hashBuf.size()); msgpack::unpacked msg; msgpack::unpack(msg, reinterpret_cast(valueBuf.data()), valueBuf.size()); @@ -152,5 +173,109 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons ); } +void +DhtProxyServer::putEncrypted(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + + int content_length = std::stoi(request->get_header("Content-Length", "0")); + + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + if(b.empty()) { + std::string response("Missing parameters"); + s->close(restbed::BAD_REQUEST, response, {{"Content-Length", std::to_string(response.size())}}); + } else { + restbed::Bytes buf(b); + + unsigned char delimiter = static_cast('\n'); + // Retrieve hash + unsigned int idx = 0; + std::vector hashBuf; + while(idx < buf.size() && buf[idx] != delimiter) { + hashBuf.emplace_back(buf[idx]); + ++idx; + } + ++idx; + // Retrieve to + std::vector toBuf; + while(idx < buf.size() && buf[idx] != delimiter) { + toBuf.emplace_back(buf[idx]); + ++idx; + } + ++idx; + // Retrieve value + std::vector valueBuf; + while(idx < buf.size()) { + valueBuf.emplace_back(buf[idx]); + ++idx; + } + InfoHash hash(hashBuf.data(), hashBuf.size()); + InfoHash to(toBuf.data(), toBuf.size()); + msgpack::unpacked msg; + msgpack::unpack(msg, reinterpret_cast(valueBuf.data()), valueBuf.size()); + auto value = std::make_shared(msg.get()); + + dht_->putEncrypted(hash, to, value); + s->close(restbed::OK, "", {{"Content-Length", "0"}}); + } + } else { + s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); + } + } + ); +} + +void +DhtProxyServer::put(const std::shared_ptr& session) const +{ + // TODO test with encrypted and signed value to send + const auto request = session->get_request(); + + int content_length = std::stoi(request->get_header("Content-Length", "0")); + + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + if(b.empty()) { + std::string response("Missing parameters"); + s->close(restbed::BAD_REQUEST, response, {{"Content-Length", std::to_string(response.size())}}); + } else { + restbed::Bytes buf(b); + + unsigned char delimiter = static_cast('\n'); + // Retrieve hash + unsigned int idx = 0; + std::vector hashBuf; + while(idx < buf.size() && buf[idx] != delimiter) { + hashBuf.emplace_back(buf[idx]); + ++idx; + } + ++idx; + // Retrieve value + std::vector valueBuf; + while(idx < buf.size()) { + valueBuf.emplace_back(buf[idx]); + ++idx; + } + // Build parameters + InfoHash hash(hashBuf.data(), hashBuf.size()); + msgpack::unpacked msg; + msgpack::unpack(msg, reinterpret_cast(valueBuf.data()), valueBuf.size()); + auto value = std::make_shared(msg.get()); + + auto response = value->toString(); + dht_->put(hash, value); + s->close(restbed::OK, response, {{"Content-Length", std::to_string(response.length())}}); + } + } else { + s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); + } + } + ); +} } From 58fdfa94520139d43ca7f716cbf83bb98e1ec813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Thu, 26 Oct 2017 16:01:45 -0400 Subject: [PATCH 066/156] add get hash endpoint --- include/opendht/dhtproxyserver.h | 5 +++ include/opendht/dhtrunner.h | 6 +++- src/dhtproxyserver.cpp | 62 +++++++++++++++++++++++++------- src/dhtrunner.cpp | 13 +++++++ tools/dhtchat.cpp | 14 +------- 5 files changed, 73 insertions(+), 27 deletions(-) diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index 0021583b..fd600341 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -26,6 +26,11 @@ public: DhtProxyServer& operator=(DhtProxyServer&& other) = default; private: + /** + * Method: GET + * PathParameter: hash + */ + void get(const std::shared_ptr& session) const; /** * Method: GET */ diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index 35f14423..b384aba8 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -364,7 +364,9 @@ public: */ void join(); - void startProxyInterface(); // TODO arguments + void startProxyInterface(); // TODO arguments and move in the constructor + void startProxyClient(const std::string& url); + void stopProxyClient(); private: static constexpr std::chrono::seconds BOOTSTRAP_PERIOD {10}; @@ -396,6 +398,8 @@ private: std::vector> rcv {}; std::unique_ptr proxy_server_; + std::atomic_bool proxyfing {false}; + /** true if currently actively boostraping */ std::atomic_bool bootstraping {false}; /* bootstrap nodes given as (host, service) pairs */ diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index 2a804a56..935cf287 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -16,6 +16,15 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) service_ = std::unique_ptr(new restbed::Service()); auto resource = std::make_shared(); + resource->set_path("/get/{hash: .*}"); + resource->set_method_handler("GET", + [this](const std::shared_ptr session) + { + this->get(session); + } + ); + service_->publish(resource); + resource = std::make_shared(); resource->set_path("/getId"); resource->set_method_handler("GET", [this](const std::shared_ptr session) @@ -81,10 +90,37 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) DhtProxyServer::~DhtProxyServer() { + service_->stop(); if (server_thread.joinable()) server_thread.join(); } +void +DhtProxyServer::get(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + dht_->get(InfoHash(hash), [s](std::shared_ptr value) { + s->yield(restbed::OK, value->toString()); + return true; + }, [s](bool ok) { + auto response = std::to_string(ok); + s->close(restbed::OK, response);//); + }); + } else { + s->close(restbed::NOT_FOUND, ""); + } + } + ); +} + void DhtProxyServer::getId(const std::shared_ptr& session) const { @@ -97,9 +133,9 @@ DhtProxyServer::getId(const std::shared_ptr& session) const { if (dht_) { auto id = dht_->getId().toString(); - s->close(restbed::OK, id, {{"Content-Length", std::to_string(id.length())}}); + s->close(restbed::OK, id); } else { - s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); + s->close(restbed::NOT_FOUND, ""); } } ); @@ -117,9 +153,9 @@ DhtProxyServer::getNodeId(const std::shared_ptr& session) cons { if (dht_) { auto id = dht_->getNodeId().toString(); - s->close(restbed::OK, id, {{"Content-Length", std::to_string(id.length())}}); + s->close(restbed::OK, id); } else { - s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); + s->close(restbed::NOT_FOUND, ""); } } ); @@ -138,7 +174,7 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons if (dht_) { if(b.empty()) { std::string response("Missing parameters"); - s->close(restbed::BAD_REQUEST, response, {{"Content-Length", std::to_string(response.size())}}); + s->close(restbed::BAD_REQUEST, response); } else { restbed::Bytes buf(b); @@ -164,10 +200,10 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons auto value = std::make_shared(msg.get()); dht_->putSigned(hash, value); - s->close(restbed::OK, "", {{"Content-Length", "0"}}); + s->close(restbed::OK, ""); } } else { - s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); + s->close(restbed::NOT_FOUND, ""); } } ); @@ -186,7 +222,7 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c if (dht_) { if(b.empty()) { std::string response("Missing parameters"); - s->close(restbed::BAD_REQUEST, response, {{"Content-Length", std::to_string(response.size())}}); + s->close(restbed::BAD_REQUEST, response); } else { restbed::Bytes buf(b); @@ -219,10 +255,10 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c auto value = std::make_shared(msg.get()); dht_->putEncrypted(hash, to, value); - s->close(restbed::OK, "", {{"Content-Length", "0"}}); + s->close(restbed::OK, ""); } } else { - s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); + s->close(restbed::NOT_FOUND, ""); } } ); @@ -242,7 +278,7 @@ DhtProxyServer::put(const std::shared_ptr& session) const if (dht_) { if(b.empty()) { std::string response("Missing parameters"); - s->close(restbed::BAD_REQUEST, response, {{"Content-Length", std::to_string(response.size())}}); + s->close(restbed::BAD_REQUEST, response); } else { restbed::Bytes buf(b); @@ -269,10 +305,10 @@ DhtProxyServer::put(const std::shared_ptr& session) const auto response = value->toString(); dht_->put(hash, value); - s->close(restbed::OK, response, {{"Content-Length", std::to_string(response.length())}}); + s->close(restbed::OK, response); } } else { - s->close(restbed::NOT_FOUND, "", {{"Content-Length", "0"}}); + s->close(restbed::NOT_FOUND, ""); } } ); diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index 16883539..c1532feb 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -807,4 +807,17 @@ DhtRunner::startProxyInterface() proxy_server_ = std::unique_ptr(new DhtProxyServer(this)); } +void +DhtRunner::startProxyClient(const std::string& url) +{ + proxyfing = true; + // TODO DhtProxyClient +} + +void +DhtRunner::stopProxyClient() +{ + proxyfing = false; +} + } diff --git a/tools/dhtchat.cpp b/tools/dhtchat.cpp index 5e80bef8..b64b4c95 100644 --- a/tools/dhtchat.cpp +++ b/tools/dhtchat.cpp @@ -20,8 +20,6 @@ #include "tools_common.h" #include -#include - extern "C" { #include } @@ -133,17 +131,7 @@ main(int argc, char **argv) std::cout << "Message publishing failed !" << std::endl; }); } else { - auto value = std::make_shared( dht::ImMessage(rand_id(rd), std::move(line), now)); - - // TODO TEMP - auto valueStr = packMsg(value); - std::cout << std::endl; - for (const auto& v: valueStr) { - std::cout << "\\x" << std::hex << static_cast(v); - } - std::cout << std::endl; - - dht.putSigned(room, value, [](bool ok) { + dht.putSigned(room, dht::ImMessage(rand_id(rd), std::move(line), now), [](bool ok) { //dht.cancelPut(room, id); if (not ok) std::cout << "Message publishing failed !" << std::endl; From 1413d32ee6f26840cf03675d460e98adc44d70a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Thu, 26 Oct 2017 17:08:53 -0400 Subject: [PATCH 067/156] add listen endpoint --- include/opendht/dhtproxyserver.h | 5 +++++ src/dhtproxyserver.cpp | 37 ++++++++++++++++++++++++++++++++ tools/dhtnode.cpp | 2 +- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index fd600341..22c6d9d7 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -39,6 +39,11 @@ private: * Method: GET */ void getNodeId(const std::shared_ptr& session) const; + /** + * Method: GET + * PathParameter: hash + */ + void listen(const std::shared_ptr& session) const; /** * Method: PUT * Body: vector of unsigned char diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index 935cf287..7fb8c8d2 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -2,12 +2,15 @@ #include "dhtproxyserver.h" +#include #include #include "default_types.h" #include "dhtrunner.h" #include "msgpack.hpp" +#include + namespace dht { DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) @@ -43,6 +46,15 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) ); service_->publish(resource); resource = std::make_shared(); + resource->set_path("/listen/{hash: .*}"); + resource->set_method_handler("GET", + [this](const std::shared_ptr session) + { + this->listen(session); + } + ); + service_->publish(resource); + resource = std::make_shared(); resource->set_path("/putSigned"); resource->set_method_handler("PUT", [this](const std::shared_ptr session) @@ -75,6 +87,8 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) settings->set_default_header("Connection", "close"); int port = 1984; int started = false; + std::chrono::seconds sec(3600); + settings->set_connection_timeout(sec); // TODO while (!started) { try { @@ -161,6 +175,29 @@ DhtProxyServer::getNodeId(const std::shared_ptr& session) cons ); } +void +DhtProxyServer::listen(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + dht_->listen(InfoHash(hash), [s](std::shared_ptr value) { + s->yield(restbed::OK, value->toString()); + return true; + }); + } else { + s->close(restbed::NOT_FOUND, ""); + } + } + ); +} + void DhtProxyServer::putSigned(const std::shared_ptr& session) const { diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 96cb394d..50255e50 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -166,7 +166,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) if (op.empty()) continue; - static const std::set VALID_OPS {"g", "l", "cl", "il", "ii", "p", "pp", "cpp", "s", "e", "a"}; + static const std::set VALID_OPS {"g", "l", "cl", "il", "ii", "p", "pp", "cpp", "s", "e", "a", "q"}; if (VALID_OPS.find(op) == VALID_OPS.cend()) { std::cout << "Unknown command: " << op << std::endl; std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl; From ca465d7159010cd6b409e30a105280326301ed8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Fri, 27 Oct 2017 14:17:32 -0400 Subject: [PATCH 068/156] set timeout to a high value + add jsoncpp --- CMakeLists.txt | 9 +++++---- src/dhtproxyserver.cpp | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 373a4c3d..a6423e4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,7 @@ include_directories ( include/ include/opendht/ ${CMAKE_CURRENT_BINARY_DIR}/include/ + /usr/include/jsoncpp ) # Install dirs @@ -179,8 +180,8 @@ if (OPENDHT_STATIC) target_link_libraries(opendht-static ${argon2_LIBRARIES}) target_include_directories(opendht-static SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - # TODO clean RESTBED link - target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} -lrestbed) + # TODO clean RESTBED link and jsoncpp + target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} -lrestbed -ljsoncpp) install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () @@ -201,8 +202,8 @@ if (OPENDHT_SHARED) target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES}) target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - # TODO clean RESTBED link - target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} -lrestbed) + # TODO clean RESTBED link and jsoncpp + target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} -lrestbed -ljsoncpp) install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index 7fb8c8d2..ad05b47f 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -10,6 +10,7 @@ #include "msgpack.hpp" #include +#include namespace dht { @@ -87,8 +88,8 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) settings->set_default_header("Connection", "close"); int port = 1984; int started = false; - std::chrono::seconds sec(3600); - settings->set_connection_timeout(sec); // TODO + std::chrono::milliseconds sec(std::numeric_limits::max()); + settings->set_connection_timeout(sec); // there is a timeout, but really huge while (!started) { try { From a3348c85d6f535308cae4a73ef16986025b5fede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Fri, 27 Oct 2017 16:33:10 -0400 Subject: [PATCH 069/156] value to JSON and begin to clean endpoints --- include/opendht/dhtproxyserver.h | 39 ++++-- include/opendht/value.h | 5 + src/dhtproxyserver.cpp | 200 +++++++++++++------------------ src/value.cpp | 30 +++++ 4 files changed, 143 insertions(+), 131 deletions(-) diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index 22c6d9d7..0495835e 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -27,23 +27,38 @@ public: private: /** - * Method: GET - * PathParameter: hash + * Return the PublicKey id, the node id and node stats + * Method: GET "/" + * Result: HTTP 200, body: {"id":"xxxx", "node_id":"xxxx", "ipv4":"xxxxxx", "ipv6": "xxxxx"} + * On error: HTTP 404, body: {"err":"xxxx"} + */ + void getNodeInfo(const std::shared_ptr& session) const; + /** + * Return Values of a InfoHash + * Method: GET "/{InfoHash: .*}" + * Return: Multiple JSON object in parts. For 2 values, you will have 3 parts: + * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * {"ok": 1} + * + * On error: HTTP 404, body: {"err":"xxxx"} */ void get(const std::shared_ptr& session) const; + /** - * Method: GET - */ - void getId(const std::shared_ptr& session) const; - /** - * Method: GET - */ - void getNodeId(const std::shared_ptr& session) const; - /** - * Method: GET - * PathParameter: hash + * Listen incoming Values of a InfoHash. + * Method: LISTEN "/{InfoHash: .*}" + * Return: Multiple JSON object in parts. For 2 values, you will have 2 parts: + * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * + * On error: HTTP 404, body: {"err":"xxxx"} */ void listen(const std::shared_ptr& session) const; + + + + /** * Method: PUT * Body: vector of unsigned char diff --git a/include/opendht/value.h b/include/opendht/value.h index 6e5a33db..f4e0c8fe 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -37,6 +37,9 @@ #include #include + +#include + namespace dht { struct Value; @@ -417,6 +420,8 @@ struct OPENDHT_PUBLIC Value return ss.str(); } + Json::Value toJson() const; + /** Return the size in bytes used by this value in memory (minimum). */ size_t size() const; diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index ad05b47f..03e3e60c 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -9,7 +9,7 @@ #include "dhtrunner.h" #include "msgpack.hpp" -#include +#include #include namespace dht { @@ -19,73 +19,36 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) // NOTE in c++14, use make_unique service_ = std::unique_ptr(new restbed::Service()); - auto resource = std::make_shared(); - resource->set_path("/get/{hash: .*}"); - resource->set_method_handler("GET", - [this](const std::shared_ptr session) - { - this->get(session); - } - ); - service_->publish(resource); - resource = std::make_shared(); - resource->set_path("/getId"); - resource->set_method_handler("GET", - [this](const std::shared_ptr session) - { - this->getId(session); - } - ); - service_->publish(resource); - resource = std::make_shared(); - resource->set_path("/getNodeId"); - resource->set_method_handler("GET", - [this](const std::shared_ptr session) - { - this->getNodeId(session); - } - ); - service_->publish(resource); - resource = std::make_shared(); - resource->set_path("/listen/{hash: .*}"); - resource->set_method_handler("GET", - [this](const std::shared_ptr session) - { - this->listen(session); - } - ); - service_->publish(resource); - resource = std::make_shared(); - resource->set_path("/putSigned"); - resource->set_method_handler("PUT", - [this](const std::shared_ptr session) - { - this->putSigned(session); - } - ); - service_->publish(resource); - resource = std::make_shared(); - resource->set_path("/putEncrypted"); - resource->set_method_handler("PUT", - [this](const std::shared_ptr session) - { - this->putEncrypted(session); - } - ); - service_->publish(resource); - resource = std::make_shared(); - resource->set_path("/put"); - resource->set_method_handler("PUT", - [this](const std::shared_ptr session) - { - this->put(session); - } - ); - service_->publish(resource); - server_thread = std::thread([this]() { + // Create endpoints + auto resource = std::make_shared(); + resource->set_path("/"); + resource->set_method_handler("GET", + [this](const std::shared_ptr session) + { + this->getNodeInfo(session); + } + ); + service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/{hash: .*}"); + resource->set_method_handler("GET", + [this](const std::shared_ptr session) + { + this->get(session); + } + ); + resource->set_method_handler("LISTEN", + [this](const std::shared_ptr session) + { + this->listen(session); + } + ); + service_->publish(resource); + + // Start server auto settings = std::make_shared(); - settings->set_default_header("Connection", "close"); + settings->set_default_header("Content-Type", "application/json"); int port = 1984; int started = false; std::chrono::milliseconds sec(std::numeric_limits::max()); @@ -110,67 +73,55 @@ DhtProxyServer::~DhtProxyServer() server_thread.join(); } +void +DhtProxyServer::getNodeInfo(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + Json::Value result; + result["id"] = dht_->getId().toString(); + result["node_id"] = dht_->getNodeId().toString(); + result["ipv4"] = dht_->getNodesStats(AF_INET).toString(); + result["ipv6"] = dht_->getNodesStats(AF_INET6).toString(); + Json::FastWriter writer; + s->close(restbed::OK, writer.write(result)); + } + else + s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); + } + ); +} + void DhtProxyServer::get(const std::shared_ptr& session) const { + static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); - int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); - session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) { if (dht_) { - dht_->get(InfoHash(hash), [s](std::shared_ptr value) { - s->yield(restbed::OK, value->toString()); + InfoHash infoHash(hash); + if (infoHash == INVALID_ID) { + infoHash = InfoHash::get(hash); + } + Json::FastWriter writer; + dht_->get(infoHash, [s, &writer](std::shared_ptr value) { + Json::Value result; + s->yield(restbed::OK, writer.write(value->toJson())); return true; - }, [s](bool ok) { + }, [s, &writer](bool ok) { auto response = std::to_string(ok); - s->close(restbed::OK, response);//); + s->close(restbed::OK, "{\"ok\": " + response + "}"); }); } else { - s->close(restbed::NOT_FOUND, ""); - } - } - ); -} - -void -DhtProxyServer::getId(const std::shared_ptr& session) const -{ - const auto request = session->get_request(); - - int content_length = std::stoi(request->get_header("Content-Length", "0")); - - session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) - { - if (dht_) { - auto id = dht_->getId().toString(); - s->close(restbed::OK, id); - } else { - s->close(restbed::NOT_FOUND, ""); - } - } - ); -} - -void -DhtProxyServer::getNodeId(const std::shared_ptr& session) const -{ - const auto request = session->get_request(); - - int content_length = std::stoi(request->get_header("Content-Length", "0")); - - session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) - { - if (dht_) { - auto id = dht_->getNodeId().toString(); - s->close(restbed::OK, id); - } else { - s->close(restbed::NOT_FOUND, ""); + s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); @@ -179,26 +130,37 @@ DhtProxyServer::getNodeId(const std::shared_ptr& session) cons void DhtProxyServer::listen(const std::shared_ptr& session) const { + static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); - int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); - + InfoHash infoHash(hash); + if (infoHash == INVALID_ID) + infoHash = InfoHash::get(hash); + size_t token; session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) { if (dht_) { - dht_->listen(InfoHash(hash), [s](std::shared_ptr value) { - s->yield(restbed::OK, value->toString()); + Json::FastWriter writer; + token = dht_->listen(infoHash, [s, &writer](std::shared_ptr value) { + Json::Value result; + s->yield(restbed::OK, writer.write(value->toJson())); return true; - }); + }).get(); } else { - s->close(restbed::NOT_FOUND, ""); + s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); + + // TODO cancel listen at the end } + + + + void DhtProxyServer::putSigned(const std::shared_ptr& session) const { diff --git a/src/value.cpp b/src/value.cpp index e3d3aece..41612bb1 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -22,6 +22,10 @@ #include "default_types.h" #include "securedht.h" // print certificate ID + +#include + + namespace dht { const std::string Query::QUERY_PARSE_ERROR {"Error parsing query."}; @@ -167,6 +171,32 @@ Value::msgpack_unpack_body(const msgpack::object& o) } } +Json::Value +Value::toJson() const +{ + // TODO check + Json::Value val; + val["id"] = std::to_string(id); + if (isEncrypted()) { + val["cypher"] = std::string(cypher.begin(), cypher.end()); + } else { + if (isSigned()) + val["sig"] = std::string(signature.begin(), signature.end()); + bool has_owner = owner && *owner; + if (has_owner) { // isSigned + val["seq"] = seq; + val["owner"] = owner->toString(); + if (recipient != InfoHash()) + val["to"] = recipient.toString(); + } + val["type"] = type; + val["data"] = std::string(data.begin(), data.end()); + if (not user_type.empty()) + val["utype"] = user_type; + } + return val; +} + bool FieldValue::operator==(const FieldValue& vfd) const { From c4ba8670e403e7063fa84f6effb9b563f88810c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Mon, 30 Oct 2017 11:43:49 -0400 Subject: [PATCH 070/156] add Putxxx value endpoints --- CMakeLists.txt | 2 + include/opendht/base64.h | 6 + include/opendht/dhtproxyserver.h | 52 ++++---- include/opendht/value.h | 40 +++++- src/base64.cpp | 95 ++++++++++++++ src/dhtproxyserver.cpp | 211 +++++++++++++++---------------- src/value.cpp | 7 +- 7 files changed, 276 insertions(+), 137 deletions(-) create mode 100644 include/opendht/base64.h create mode 100644 src/base64.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a6423e4b..18cd7322 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,6 +105,7 @@ list (APPEND opendht_SOURCES src/node.cpp src/value.cpp src/dht.cpp + src/base64.cpp src/dhtproxyserver.cpp src/storage.h src/listener.h @@ -133,6 +134,7 @@ list (APPEND opendht_HEADERS include/opendht/node.h include/opendht/value.h include/opendht/dht.h + include/opendht/base64.h include/opendht/dhtproxyserver.h include/opendht/callbacks.h include/opendht/routing_table.h diff --git a/include/opendht/base64.h b/include/opendht/base64.h new file mode 100644 index 00000000..256e2640 --- /dev/null +++ b/include/opendht/base64.h @@ -0,0 +1,6 @@ +#ifndef _BASE64_H_ +#define _BASE64_H_ +#include +std::string base64_encode(unsigned char const* , unsigned int len); +std::string base64_decode(std::string const& s); +#endif diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index 0495835e..94b02a48 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -33,6 +33,7 @@ private: * On error: HTTP 404, body: {"err":"xxxx"} */ void getNodeInfo(const std::shared_ptr& session) const; + /** * Return Values of a InfoHash * Method: GET "/{InfoHash: .*}" @@ -56,35 +57,36 @@ private: */ void listen(const std::shared_ptr& session) const; - - - /** - * Method: PUT - * Body: vector of unsigned char - * first line: InfoHash hash - * second line: Value value to sign and put - * @param session - */ - void putSigned(const std::shared_ptr& session) const; - /** - * Method: PUT - * Body: vector of unsigned char - * first line: InfoHash hash - * second line: InfoHash to - * third line: Value value to encrypt and put - * @param session - */ - void putEncrypted(const std::shared_ptr& session) const; - /** - * Method: PUT - * Body: vector of unsigned char - * first line: InfoHash hash - * second line: Value value to put - * @param session + * Put a value on the DHT + * Method: POST "/{InfoHash: .*}" + * body = Value to put in JSON + * Return: {"ok":"1"} + * On error: HTTP 404, body: {"err":"xxxx"} if no dht + * HTTP 400, body: {"err":"xxxx"} if bad json */ void put(const std::shared_ptr& session) const; + /** + * Put a value to sign by the proxy on the DHT + * Method: SIGN "/{InfoHash: .*}" + * body = Value to put in JSON + * Return: {"ok":"1"} + * On error: HTTP 404, body: {"err":"xxxx"} if no dht + * HTTP 400, body: {"err":"xxxx"} if bad json + */ + void putSigned(const std::shared_ptr& session) const; + + /** + * Put a value to encrypt by the proxy on the DHT + * Method: ENCRYPT "/{InfoHash: .*}" + * body = Value to put in JSON + * Return: {"ok":"1"} + * On error: HTTP 404, body: {"err":"xxxx"} if no dht + * HTTP 400, body: {"err":"xxxx"} if bad json + */ + void putEncrypted(const std::shared_ptr& session) const; + std::thread server_thread {}; std::unique_ptr service_; DhtRunner* dht_; diff --git a/include/opendht/value.h b/include/opendht/value.h index f4e0c8fe..9113805b 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -19,6 +19,7 @@ #pragma once +#include "base64.h" #include "infohash.h" #include "crypto.h" #include "utils.h" @@ -37,7 +38,6 @@ #include #include - #include namespace dht { @@ -349,6 +349,44 @@ struct OPENDHT_PUBLIC Value Value(ValueType::Id t, const uint8_t* dat_ptr, size_t dat_len, Id id = INVALID_ID) : id(id), type(t), data(dat_ptr, dat_ptr+dat_len) {} + Value(Json::Value& json) { + + try { + if (json.isMember("id")) + id = ValueType::Id(json["id"].asInt()); + } catch (...) { } + if (json.isMember("cypher")) { + auto cypherStr = json["cypher"].asString(); + cypherStr = base64_decode(cypherStr); + cypher = std::vector(cypherStr.begin(), cypherStr.end()); + } + if (json.isMember("sig")) { + auto sigStr = json["sig"].asString(); + sigStr = base64_decode(sigStr); + signature = std::vector(sigStr.begin(), sigStr.end()); + } + if (json.isMember("seq")) + seq = json["seq"].asInt(); + if (json.isMember("owner")) { + auto ownerStr = json["owner"].asString(); + auto ownerBlob = std::vector(ownerStr.begin(), ownerStr.end()); + owner = std::make_shared(ownerBlob); + } + if (json.isMember("to")) { + auto toStr = json["to"].asString(); + recipient = InfoHash(toStr); + } + if (json.isMember("type")) + type = json["type"].asInt(); + if (json.isMember("data")){ + auto dataStr = json["data"].asString(); + dataStr = base64_decode(dataStr); + data = std::vector(dataStr.begin(), dataStr.end()); + } + if (json.isMember("utype")) + user_type = json["utype"].asString(); + } + template Value(ValueType::Id t, const Type& d, Id id = INVALID_ID) : id(id), type(t), data(packMsg(d)) {} diff --git a/src/base64.cpp b/src/base64.cpp new file mode 100644 index 00000000..d11c28f2 --- /dev/null +++ b/src/base64.cpp @@ -0,0 +1,95 @@ +#include "base64.h" + +static const std::string base64_chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + + +static inline bool is_base64(unsigned char c) { + return (isalnum(c) || (c == '+') || (c == '/')); +} + +std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { + std::string ret; + int i = 0; + int j = 0; + unsigned char char_array_3[3]; + unsigned char char_array_4[4]; + + while (in_len--) { + char_array_3[i++] = *(bytes_to_encode++); + if (i == 3) { + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; + + for(i = 0; (i <4) ; i++) + ret += base64_chars[char_array_4[i]]; + i = 0; + } + } + + if (i) + { + for(j = i; j < 3; j++) + char_array_3[j] = '\0'; + + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; + + for (j = 0; (j < i + 1); j++) + ret += base64_chars[char_array_4[j]]; + + while((i++ < 3)) + ret += '='; + + } + + return ret; + +} + +std::string base64_decode(std::string const& encoded_string) { + size_t in_len = encoded_string.size(); + size_t i = 0; + size_t j = 0; + int in_ = 0; + unsigned char char_array_4[4], char_array_3[3]; + std::string ret; + + while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { + char_array_4[i++] = encoded_string[in_]; in_++; + if (i ==4) { + for (i = 0; i <4; i++) + char_array_4[i] = static_cast(base64_chars.find(char_array_4[i])); + + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (i = 0; (i < 3); i++) + ret += char_array_3[i]; + i = 0; + } + } + + if (i) { + for (j = i; j <4; j++) + char_array_4[j] = 0; + + for (j = 0; j <4; j++) + char_array_4[j] = static_cast(base64_chars.find(char_array_4[j])); + + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; + } + + return ret; +} diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index 03e3e60c..ea8ea0af 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace dht { DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) @@ -44,6 +46,27 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) this->listen(session); } ); + resource->set_method_handler("POST", + [this](const std::shared_ptr session) + { + this->put(session); + } + ); + resource->set_method_handler("SIGN", + [this](const std::shared_ptr session) + { + this->putSigned(session); + } + ); + service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/{hash: .*}/{hash: .*}"); + resource->set_method_handler("ENCRYPT", + [this](const std::shared_ptr session) + { + this->putEncrypted(session); + } + ); service_->publish(resource); // Start server @@ -51,7 +74,8 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) settings->set_default_header("Content-Type", "application/json"); int port = 1984; int started = false; - std::chrono::milliseconds sec(std::numeric_limits::max()); + std::chrono::seconds sec(3600); + //std::chrono::milliseconds sec(std::numeric_limits::max()); settings->set_connection_timeout(sec); // there is a timeout, but really huge while (!started) { @@ -153,20 +177,21 @@ DhtProxyServer::listen(const std::shared_ptr& session) const } } ); - // TODO cancel listen at the end } - - - void -DhtProxyServer::putSigned(const std::shared_ptr& session) const +DhtProxyServer::put(const std::shared_ptr& session) const { + // TODO test with encrypted and signed value to send + static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); - int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash infoHash(hash); + if (infoHash == INVALID_ID) + infoHash = InfoHash::get(hash); session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) @@ -177,30 +202,60 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons s->close(restbed::BAD_REQUEST, response); } else { restbed::Bytes buf(b); + Json::Value root; + Json::Reader reader; + std::string strJson(buf.begin(), buf.end()); + bool parsingSuccessful = reader.parse(strJson.c_str(), root); + if (parsingSuccessful) { + auto value = std::make_shared(root); - unsigned char delimiter = static_cast('\n'); - // Retrieve hash - unsigned int idx = 0; - std::vector hashBuf; - while(idx < buf.size() && buf[idx] != delimiter) { - hashBuf.emplace_back(buf[idx]); - ++idx; + auto response = value->toString(); + dht_->put(infoHash, value); + s->close(restbed::OK, response); + } else { + s->close(restbed::BAD_REQUEST, "Incorrect JSON"); } - ++idx; - // Retrieve value - std::vector valueBuf; - while(idx < buf.size()) { - valueBuf.emplace_back(buf[idx]); - ++idx; - } - // Build parameters - InfoHash hash(hashBuf.data(), hashBuf.size()); - msgpack::unpacked msg; - msgpack::unpack(msg, reinterpret_cast(valueBuf.data()), valueBuf.size()); - auto value = std::make_shared(msg.get()); + } + } else { + s->close(restbed::NOT_FOUND, ""); + } + } + ); +} - dht_->putSigned(hash, value); - s->close(restbed::OK, ""); +void +DhtProxyServer::putSigned(const std::shared_ptr& session) const +{ + static constexpr dht::InfoHash INVALID_ID {}; + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash infoHash(hash); + if (infoHash == INVALID_ID) + infoHash = InfoHash::get(hash); + + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + if(b.empty()) { + std::string response("Missing parameters"); + s->close(restbed::BAD_REQUEST, response); + } else { + restbed::Bytes buf(b); + Json::Value root; + Json::Reader reader; + std::string strJson(buf.begin(), buf.end()); + bool parsingSuccessful = reader.parse(strJson.c_str(), root); + if (parsingSuccessful) { + auto value = std::make_shared(root); + + auto response = value->toString(); + dht_->putSigned(infoHash, value); + s->close(restbed::OK, response); + } else { + s->close(restbed::BAD_REQUEST, "Incorrect JSON" + strJson); + } } } else { s->close(restbed::NOT_FOUND, ""); @@ -212,9 +267,17 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons void DhtProxyServer::putEncrypted(const std::shared_ptr& session) const { + static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); - int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash infoHash(hash); + if (infoHash == INVALID_ID) + infoHash = InfoHash::get(hash); + auto toHash = request->get_path_parameter("to"); + InfoHash toInfoHash(toHash); + if (toInfoHash == INVALID_ID) + toInfoHash = InfoHash::get(toHash); session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) @@ -225,87 +288,19 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c s->close(restbed::BAD_REQUEST, response); } else { restbed::Bytes buf(b); + Json::Value root; + Json::Reader reader; + std::string strJson(buf.begin(), buf.end()); + bool parsingSuccessful = reader.parse(strJson.c_str(), root); + if (parsingSuccessful) { + auto value = std::make_shared(root); - unsigned char delimiter = static_cast('\n'); - // Retrieve hash - unsigned int idx = 0; - std::vector hashBuf; - while(idx < buf.size() && buf[idx] != delimiter) { - hashBuf.emplace_back(buf[idx]); - ++idx; + auto response = value->toString(); + dht_->putEncrypted(infoHash, toInfoHash, value); + s->close(restbed::OK, response); + } else { + s->close(restbed::BAD_REQUEST, "Incorrect JSON"); } - ++idx; - // Retrieve to - std::vector toBuf; - while(idx < buf.size() && buf[idx] != delimiter) { - toBuf.emplace_back(buf[idx]); - ++idx; - } - ++idx; - // Retrieve value - std::vector valueBuf; - while(idx < buf.size()) { - valueBuf.emplace_back(buf[idx]); - ++idx; - } - InfoHash hash(hashBuf.data(), hashBuf.size()); - InfoHash to(toBuf.data(), toBuf.size()); - msgpack::unpacked msg; - msgpack::unpack(msg, reinterpret_cast(valueBuf.data()), valueBuf.size()); - auto value = std::make_shared(msg.get()); - - dht_->putEncrypted(hash, to, value); - s->close(restbed::OK, ""); - } - } else { - s->close(restbed::NOT_FOUND, ""); - } - } - ); -} - -void -DhtProxyServer::put(const std::shared_ptr& session) const -{ - // TODO test with encrypted and signed value to send - const auto request = session->get_request(); - - int content_length = std::stoi(request->get_header("Content-Length", "0")); - - session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) - { - if (dht_) { - if(b.empty()) { - std::string response("Missing parameters"); - s->close(restbed::BAD_REQUEST, response); - } else { - restbed::Bytes buf(b); - - unsigned char delimiter = static_cast('\n'); - // Retrieve hash - unsigned int idx = 0; - std::vector hashBuf; - while(idx < buf.size() && buf[idx] != delimiter) { - hashBuf.emplace_back(buf[idx]); - ++idx; - } - ++idx; - // Retrieve value - std::vector valueBuf; - while(idx < buf.size()) { - valueBuf.emplace_back(buf[idx]); - ++idx; - } - // Build parameters - InfoHash hash(hashBuf.data(), hashBuf.size()); - msgpack::unpacked msg; - msgpack::unpack(msg, reinterpret_cast(valueBuf.data()), valueBuf.size()); - auto value = std::make_shared(msg.get()); - - auto response = value->toString(); - dht_->put(hash, value); - s->close(restbed::OK, response); } } else { s->close(restbed::NOT_FOUND, ""); diff --git a/src/value.cpp b/src/value.cpp index 41612bb1..156a7938 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -24,6 +24,7 @@ #include +#include namespace dht { @@ -178,10 +179,10 @@ Value::toJson() const Json::Value val; val["id"] = std::to_string(id); if (isEncrypted()) { - val["cypher"] = std::string(cypher.begin(), cypher.end()); + val["cypher"] = base64_encode(cypher.data(), cypher.size()); } else { if (isSigned()) - val["sig"] = std::string(signature.begin(), signature.end()); + val["sig"] = base64_encode(signature.data(), signature.size()); bool has_owner = owner && *owner; if (has_owner) { // isSigned val["seq"] = seq; @@ -190,7 +191,7 @@ Value::toJson() const val["to"] = recipient.toString(); } val["type"] = type; - val["data"] = std::string(data.begin(), data.end()); + val["data"] = base64_encode(data.data(), data.size()); if (not user_type.empty()) val["utype"] = user_type; } From 20abc3e4957493734eef9af9177ed220902b6a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Mon, 30 Oct 2017 13:54:10 -0400 Subject: [PATCH 071/156] add LISTEN endpoint --- include/opendht/dhtproxyserver.h | 1 + src/dhtproxyserver.cpp | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index 94b02a48..bda4fa8c 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -90,6 +90,7 @@ private: std::thread server_thread {}; std::unique_ptr service_; DhtRunner* dht_; + mutable std::vector> listenThreads_; }; } diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index ea8ea0af..21a9cc42 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -93,6 +93,10 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) DhtProxyServer::~DhtProxyServer() { service_->stop(); + // listenThreads_ will stop because there is no more sessions + for (auto& listenThread: listenThreads_) + if (listenThread->joinable()) + listenThread->join(); if (server_thread.joinable()) server_thread.join(); } @@ -169,15 +173,24 @@ DhtProxyServer::listen(const std::shared_ptr& session) const Json::FastWriter writer; token = dht_->listen(infoHash, [s, &writer](std::shared_ptr value) { Json::Value result; - s->yield(restbed::OK, writer.write(value->toJson())); - return true; + if (!s->is_closed()) + s->yield(restbed::OK, writer.write(value->toJson())); + return !s->is_closed(); }).get(); + // Handle client deconnection + listenThreads_.emplace_back(std::shared_ptr( + new std::thread([&]() { + while(!s->is_closed()) + std::this_thread::sleep_for(std::chrono::seconds(2)); + if (dht_) + dht_->cancelListen(infoHash, token); + })) + ); } else { s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); - // TODO cancel listen at the end } From 7ddd6629e36cdd83229a68e41aab73282c87ddee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Tue, 31 Oct 2017 15:24:52 -0400 Subject: [PATCH 072/156] add getFiltered --- include/opendht/dhtproxyserver.h | 16 +++++++-- src/dhtproxyserver.cpp | 61 +++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index bda4fa8c..ac4f5a6e 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -79,14 +79,26 @@ private: /** * Put a value to encrypt by the proxy on the DHT - * Method: ENCRYPT "/{InfoHash: .*}" - * body = Value to put in JSON + * Method: ENCRYPT "/{hash: .*}" + * body = Value to put in JSON + "to":"infoHash" * Return: {"ok":"1"} * On error: HTTP 404, body: {"err":"xxxx"} if no dht * HTTP 400, body: {"err":"xxxx"} if bad json */ void putEncrypted(const std::shared_ptr& session) const; + /** + * Return Values of a InfoHash filtered by a value id + * Method: GET "/{InfoHash: .*}/{ValueId: .*}" + * Return: Multiple JSON object in parts. For 2 values, you will have 3 parts: + * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * {"ok": 1} + * + * On error: HTTP 404, body: {"err":"xxxx"} + */ + void getFiltered(const std::shared_ptr& session) const; + std::thread server_thread {}; std::unique_ptr service_; DhtRunner* dht_; diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index 21a9cc42..13f4fe67 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -52,15 +52,13 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) this->put(session); } ); + // NOTE/TODO ENCRYPT AND SIGN must be optionnal resource->set_method_handler("SIGN", [this](const std::shared_ptr session) { this->putSigned(session); } ); - service_->publish(resource); - resource = std::make_shared(); - resource->set_path("/{hash: .*}/{hash: .*}"); resource->set_method_handler("ENCRYPT", [this](const std::shared_ptr session) { @@ -68,6 +66,15 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) } ); service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/{hash: .*}/{value: .*}"); + resource->set_method_handler("GET", + [this](const std::shared_ptr session) + { + this->getFiltered(session); + } + ); + service_->publish(resource); // Start server auto settings = std::make_shared(); @@ -287,10 +294,6 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c InfoHash infoHash(hash); if (infoHash == INVALID_ID) infoHash = InfoHash::get(hash); - auto toHash = request->get_path_parameter("to"); - InfoHash toInfoHash(toHash); - if (toInfoHash == INVALID_ID) - toInfoHash = InfoHash::get(toHash); session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) @@ -305,14 +308,22 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c Json::Reader reader; std::string strJson(buf.begin(), buf.end()); bool parsingSuccessful = reader.parse(strJson.c_str(), root); - if (parsingSuccessful) { + auto toHash = root["to"].asString(); + if (parsingSuccessful && !toHash.empty()) { auto value = std::make_shared(root); + auto toHash = request->get_path_parameter("to"); + InfoHash toInfoHash(toHash); + if (toInfoHash == INVALID_ID) + toInfoHash = InfoHash::get(toHash); auto response = value->toString(); dht_->putEncrypted(infoHash, toInfoHash, value); s->close(restbed::OK, response); } else { - s->close(restbed::BAD_REQUEST, "Incorrect JSON"); + if(!parsingSuccessful) + s->close(restbed::BAD_REQUEST, "Incorrect JSON"); + else + s->close(restbed::BAD_REQUEST, "No destination found"); } } } else { @@ -322,4 +333,36 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c ); } +void +DhtProxyServer::getFiltered(const std::shared_ptr& session) const +{ + static constexpr dht::InfoHash INVALID_ID {}; + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + auto value = request->get_path_parameter("value"); + session->fetch(content_length, + [&](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + InfoHash infoHash(hash); + if (infoHash == INVALID_ID) { + infoHash = InfoHash::get(hash); + } + Json::FastWriter writer; + dht_->get(infoHash, [s, &writer](std::shared_ptr value) { + Json::Value result; + s->yield(restbed::OK, writer.write(value->toJson())); + return true; + }, [s, &writer](bool ok) { + auto response = std::to_string(ok); + s->close(restbed::OK, "{\"ok\": " + response + "}"); + }, {}, value); + } else { + s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); + } + } + ); +} + } From d608da4307dec7555f50ebb6fe58f837383eb002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 1 Nov 2017 10:03:01 -0400 Subject: [PATCH 073/156] cmake: add optionnal dependencies for proxy --- CMakeLists.txt | 19 +++++++++++++++---- cmake/FindJsoncpp.cmake | 18 ++++++++++++++++++ cmake/FindRestbed.cmake | 18 ++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 cmake/FindJsoncpp.cmake create mode 100644 cmake/FindRestbed.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 18cd7322..171a1c7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,9 @@ option (OPENDHT_SYSTEMD "Install systemd module" OFF) option (OPENDHT_ARGON2 "Use included argon2 sources" OFF) option (OPENDHT_LTO "Build with LTO" OFF) option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF) +option (OPENDHT_PROXY_SERVER "Enable DHT proxy server, use Restbed and jsoncpp" OFF) +option (OPENDHT_PROXY_SERVER_OPTIONAL "Enable sign and encrypt endpoints" OFF) +option (OPENDHT_PROXY_CLIENT "Enable DHT proxy client, use Restbed and jsoncpp" OFF) find_package(Doxygen) option (OPENDHT_DOCUMENTATION "Create and install the HTML based API documentation (requires Doxygen)" ${DOXYGEN_FOUND}) @@ -41,6 +44,10 @@ if (NOT OPENDHT_ARGON2) set(OPENDHT_ARGON2 ON) endif() endif () +if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT) + find_package(Restbed REQUIRED) + find_package(Jsoncpp REQUIRED) +endif() # Build flags set (CMAKE_CXX_STANDARD 11) @@ -77,6 +84,12 @@ endif () if (Nettle_INCLUDE_DIRS) include_directories (SYSTEM "${Nettle_INCLUDE_DIRS}") endif () +if (Restbed_INCLUDE_DIRS) + include_directories (SYSTEM "${Restbed_INCLUDE_DIRS}") +endif () +if (Jsoncpp_INCLUDE_DIRS) + include_directories (SYSTEM "${Jsoncpp_INCLUDE_DIRS}") +endif () link_directories (${Nettle_LIBRARY_DIRS}) include_directories ( ./ @@ -182,8 +195,7 @@ if (OPENDHT_STATIC) target_link_libraries(opendht-static ${argon2_LIBRARIES}) target_include_directories(opendht-static SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - # TODO clean RESTBED link and jsoncpp - target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} -lrestbed -ljsoncpp) + target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${Restbed_LIBRARIES} ${Jsoncpp_LIBRARIES}) install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () @@ -204,8 +216,7 @@ if (OPENDHT_SHARED) target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES}) target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - # TODO clean RESTBED link and jsoncpp - target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} -lrestbed -ljsoncpp) + target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${Restbed_LIBRARIES} ${Jsoncpp_LIBRARIES}) install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () diff --git a/cmake/FindJsoncpp.cmake b/cmake/FindJsoncpp.cmake new file mode 100644 index 00000000..fccc2da3 --- /dev/null +++ b/cmake/FindJsoncpp.cmake @@ -0,0 +1,18 @@ +find_path (JSONCPP_INCLUDE jsoncpp + HINTS + "/usr/include" + "/usr/local/include" + "/opt/local/include" +) + +if (JSONCPP_INCLUDE) + message(STATUS "${green}Found Jsoncpp: ${JSONCPP_INCLUDE}") +else() + message(FATAL_ERROR "${red}Failed to locate Jsoncpp.}") +endif() + +if (JSONCPP_INCLUDE) + set(JSONCPP_FOUND TRUE) + set(Jsoncpp_LIBRARIES jsoncpp) + set(Jsoncpp_INCLUDE_DIRS ${JSONCPP_INCLUDE}/jsoncpp) +endif() diff --git a/cmake/FindRestbed.cmake b/cmake/FindRestbed.cmake new file mode 100644 index 00000000..440e5596 --- /dev/null +++ b/cmake/FindRestbed.cmake @@ -0,0 +1,18 @@ +find_path (RESTBED_INCLUDE restbed + HINTS + "/usr/include" + "/usr/local/include" + "/opt/local/include" +) + +if (RESTBED_INCLUDE) + message(STATUS "${green}Found Restbed: ${RESTBED_INCLUDE}") +else() + message(FATAL_ERROR "${red}Failed to locate Restbed.}") +endif() + +if (RESTBED_INCLUDE) + set(RESTBED_FOUND TRUE) + set(Restbed_LIBRARIES restbed) + set(Restbed_INCLUDE_DIRS ${RESTBED_INCLUDE}/restbed) +endif() From dc66b3847a834eb40ba16e563724a0a95131c435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 1 Nov 2017 11:00:00 -0400 Subject: [PATCH 074/156] make code for DHT Proxy Server optionnal --- CMakeLists.txt | 18 ++++++++++++++---- include/opendht/base64.h | 8 +++++--- include/opendht/dhtproxyserver.h | 6 ++++-- include/opendht/dhtrunner.h | 9 ++++++--- include/opendht/value.h | 8 +++++++- src/base64.cpp | 2 ++ src/dhtproxyserver.cpp | 2 ++ src/dhtrunner.cpp | 15 ++------------- src/value.cpp | 6 ++++-- 9 files changed, 46 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 171a1c7d..30a934a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,8 +118,6 @@ list (APPEND opendht_SOURCES src/node.cpp src/value.cpp src/dht.cpp - src/base64.cpp - src/dhtproxyserver.cpp src/storage.h src/listener.h src/search.h @@ -147,8 +145,6 @@ list (APPEND opendht_HEADERS include/opendht/node.h include/opendht/value.h include/opendht/dht.h - include/opendht/base64.h - include/opendht/dhtproxyserver.h include/opendht/callbacks.h include/opendht/routing_table.h include/opendht/node_cache.h @@ -162,6 +158,20 @@ list (APPEND opendht_HEADERS include/opendht/indexation/pht.h ) +if (OPENDHT_PROXY_SERVER) + add_definitions(-DOPENDHT_PROXY_SERVER=true) + list (APPEND opendht_HEADERS + include/opendht/dhtproxyserver.h + include/opendht/base64.h + ) + list (APPEND opendht_SOURCES + src/dhtproxyserver.cpp + src/base64.cpp + ) +else () + add_definitions(-DENABLE_PROXY_SERVER=false) +endif () + if(OPENDHT_ARGON2) # make sure argon2 submodule is up to date and initialized message("Initializing Argon2 submodule") diff --git a/include/opendht/base64.h b/include/opendht/base64.h index 256e2640..8f4d9ebb 100644 --- a/include/opendht/base64.h +++ b/include/opendht/base64.h @@ -1,6 +1,8 @@ -#ifndef _BASE64_H_ -#define _BASE64_H_ +#if OPENDHT_PROXY_SERVER +#pragma once + #include + std::string base64_encode(unsigned char const* , unsigned int len); std::string base64_decode(std::string const& s); -#endif +#endif // OPENDHT_PROXY_SERVER diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index ac4f5a6e..380db0f5 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -1,13 +1,13 @@ // TODO gpl +#if OPENDHT_PROXY_SERVER + #pragma once #include "def.h" #include #include - -// TODO add ifdef RESTBED #include namespace dht { @@ -106,3 +106,5 @@ private: }; } + +#endif //OPENDHT_PROXY_SERVER diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index b384aba8..e2237fbf 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -36,7 +36,9 @@ #include #include +#if OPENDHT_PROXY_SERVER #include "dhtproxyserver.h" +#endif namespace dht { @@ -364,9 +366,9 @@ public: */ void join(); +#if OPENDHT_PROXY_SERVER void startProxyInterface(); // TODO arguments and move in the constructor - void startProxyClient(const std::string& url); - void stopProxyClient(); +#endif //OPENDHT_PROXY_SERVER private: static constexpr std::chrono::seconds BOOTSTRAP_PERIOD {10}; @@ -397,8 +399,9 @@ private: std::mutex sock_mtx {}; std::vector> rcv {}; +#if OPENDHT_PROXY_SERVER std::unique_ptr proxy_server_; - std::atomic_bool proxyfing {false}; +#endif //OPENDHT_PROXY_SERVER /** true if currently actively boostraping */ std::atomic_bool bootstraping {false}; diff --git a/include/opendht/value.h b/include/opendht/value.h index 9113805b..4119d7ab 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -19,7 +19,6 @@ #pragma once -#include "base64.h" #include "infohash.h" #include "crypto.h" #include "utils.h" @@ -38,7 +37,10 @@ #include #include +#if OPENDHT_PROXY_SERVER #include +#include "base64.h" +#endif //OPENDHT_PROXY_SERVER namespace dht { @@ -349,6 +351,7 @@ struct OPENDHT_PUBLIC Value Value(ValueType::Id t, const uint8_t* dat_ptr, size_t dat_len, Id id = INVALID_ID) : id(id), type(t), data(dat_ptr, dat_ptr+dat_len) {} +#if OPENDHT_PROXY_SERVER Value(Json::Value& json) { try { @@ -386,6 +389,7 @@ struct OPENDHT_PUBLIC Value if (json.isMember("utype")) user_type = json["utype"].asString(); } +#endif //OPENDHT_PROXY_SERVER template Value(ValueType::Id t, const Type& d, Id id = INVALID_ID) @@ -458,7 +462,9 @@ struct OPENDHT_PUBLIC Value return ss.str(); } +#if OPENDHT_PROXY_SERVER Json::Value toJson() const; +#endif //OPENDHT_PROXY_SERVER /** Return the size in bytes used by this value in memory (minimum). */ size_t size() const; diff --git a/src/base64.cpp b/src/base64.cpp index d11c28f2..4068c003 100644 --- a/src/base64.cpp +++ b/src/base64.cpp @@ -1,3 +1,4 @@ +#if OPENDHT_PROXY_SERVER #include "base64.h" static const std::string base64_chars = @@ -93,3 +94,4 @@ std::string base64_decode(std::string const& encoded_string) { return ret; } +#endif // OPENDHT_PROXY_SERVER diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index 13f4fe67..e297d02d 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -1,5 +1,6 @@ // TODO GPL +#if OPENDHT_PROXY_SERVER #include "dhtproxyserver.h" #include @@ -366,3 +367,4 @@ DhtProxyServer::getFiltered(const std::shared_ptr& session) co } } +#endif //OPENDHT_PROXY_SERVER diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index c1532feb..8019ef73 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -800,24 +800,13 @@ DhtRunner::findCertificate(InfoHash hash, std::function(new DhtProxyServer(this)); } - -void -DhtRunner::startProxyClient(const std::string& url) -{ - proxyfing = true; - // TODO DhtProxyClient -} - -void -DhtRunner::stopProxyClient() -{ - proxyfing = false; -} +#endif //OPENDHT_PROXY_SERVER } diff --git a/src/value.cpp b/src/value.cpp index 156a7938..1565ce8d 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -22,9 +22,9 @@ #include "default_types.h" #include "securedht.h" // print certificate ID - -#include +#if OPENDHT_PROXY_SERVER #include +#endif //OPENDHT_PROXY_SERVER namespace dht { @@ -172,6 +172,7 @@ Value::msgpack_unpack_body(const msgpack::object& o) } } +#if OPENDHT_PROXY_SERVER Json::Value Value::toJson() const { @@ -197,6 +198,7 @@ Value::toJson() const } return val; } +#endif //OPENDHT_PROXY_SERVER bool FieldValue::operator==(const FieldValue& vfd) const From 8344138bc8c0c1f680ded99b2669b4f72ef58548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 1 Nov 2017 12:25:38 -0400 Subject: [PATCH 075/156] add optional + clean start/stop --- CMakeLists.txt | 5 +++++ include/opendht/dhtproxyserver.h | 6 +++++- include/opendht/dhtrunner.h | 3 ++- src/dhtproxyserver.cpp | 36 +++++++++++++++++--------------- src/dhtrunner.cpp | 11 ++++++++-- 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 30a934a0..c94b818f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,6 +160,11 @@ list (APPEND opendht_HEADERS if (OPENDHT_PROXY_SERVER) add_definitions(-DOPENDHT_PROXY_SERVER=true) + if (OPENDHT_PROXY_SERVER_OPTIONAL) + add_definitions(-DOPENDHT_PROXY_SERVER_OPTIONAL=true) + else () + add_definitions(-DOPENDHT_PROXY_SERVER_OPTIONAL=false) + endif() list (APPEND opendht_HEADERS include/opendht/dhtproxyserver.h include/opendht/base64.h diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index 380db0f5..17f13f9a 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -17,7 +17,7 @@ class DhtRunner; class OPENDHT_PUBLIC DhtProxyServer { public: - DhtProxyServer(DhtRunner* dht); + DhtProxyServer(DhtRunner* dht, unsigned int port = 8000); virtual ~DhtProxyServer(); DhtProxyServer(const DhtProxyServer& other) = default; @@ -25,6 +25,8 @@ public: DhtProxyServer& operator=(const DhtProxyServer& other) = default; DhtProxyServer& operator=(DhtProxyServer&& other) = default; + void stop(); + private: /** * Return the PublicKey id, the node id and node stats @@ -67,6 +69,7 @@ private: */ void put(const std::shared_ptr& session) const; +#if OPENDHT_PROXY_SERVER_OPTIONAL /** * Put a value to sign by the proxy on the DHT * Method: SIGN "/{InfoHash: .*}" @@ -86,6 +89,7 @@ private: * HTTP 400, body: {"err":"xxxx"} if bad json */ void putEncrypted(const std::shared_ptr& session) const; +#endif // OPENDHT_PROXY_SERVER_OPTIONAL /** * Return Values of a InfoHash filtered by a value id diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index e2237fbf..8e6cfd6b 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -367,7 +367,8 @@ public: void join(); #if OPENDHT_PROXY_SERVER - void startProxyInterface(); // TODO arguments and move in the constructor + void startProxyInterface(unsigned int port=8000); + void stopProxyInterface(); #endif //OPENDHT_PROXY_SERVER private: diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index e297d02d..de35c8cf 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -17,12 +17,12 @@ namespace dht { -DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) +DhtProxyServer::DhtProxyServer(DhtRunner* dht, unsigned int port) : dht_(dht) { // NOTE in c++14, use make_unique service_ = std::unique_ptr(new restbed::Service()); - server_thread = std::thread([this]() { + server_thread = std::thread([this, port]() { // Create endpoints auto resource = std::make_shared(); resource->set_path("/"); @@ -53,7 +53,7 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) this->put(session); } ); - // NOTE/TODO ENCRYPT AND SIGN must be optionnal +#if OPENDHT_PROXY_SERVER_OPTIONAL resource->set_method_handler("SIGN", [this](const std::shared_ptr session) { @@ -66,6 +66,7 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) this->putEncrypted(session); } ); +#endif // OPENDHT_PROXY_SERVER_OPTIONAL service_->publish(resource); resource = std::make_shared(); resource->set_path("/{hash: .*}/{value: .*}"); @@ -80,25 +81,24 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht) : dht_(dht) // Start server auto settings = std::make_shared(); settings->set_default_header("Content-Type", "application/json"); - int port = 1984; - int started = false; - std::chrono::seconds sec(3600); - //std::chrono::milliseconds sec(std::numeric_limits::max()); - settings->set_connection_timeout(sec); // there is a timeout, but really huge - while (!started) - { - try { - settings->set_port(port); // TODO add argument and move this - service_->start(settings); - started = true; - } catch (std::system_error& e) { - port += 1; - } + std::chrono::milliseconds timeout(std::numeric_limits::max()); + settings->set_connection_timeout(timeout); // there is a timeout, but really huge + settings->set_port(port); + try { + service_->start(settings); + } catch(std::system_error& e) { + // Fail silently for now. } }); } DhtProxyServer::~DhtProxyServer() +{ + stop(); +} + +void +DhtProxyServer::stop() { service_->stop(); // listenThreads_ will stop because there is no more sessions @@ -244,6 +244,7 @@ DhtProxyServer::put(const std::shared_ptr& session) const ); } +#if OPENDHT_PROXY_SERVER_OPTIONAL void DhtProxyServer::putSigned(const std::shared_ptr& session) const { @@ -333,6 +334,7 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c } ); } +#endif // OPENDHT_PROXY_SERVER_OPTIONAL void DhtProxyServer::getFiltered(const std::shared_ptr& session) const diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index 8019ef73..c33ba231 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -802,10 +802,17 @@ DhtRunner::findCertificate(InfoHash hash, std::function(new DhtProxyServer(this)); + proxy_server_ = std::unique_ptr(new DhtProxyServer(this, port)); +} + +void +DhtRunner::stopProxyInterface() +{ + if (proxy_server_) + proxy_server_.reset(nullptr); } #endif //OPENDHT_PROXY_SERVER From 8e43775394d8a74709384c567c4fd3520f32f84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 1 Nov 2017 14:04:09 -0400 Subject: [PATCH 076/156] update dhtnode + clean code --- CMakeLists.txt | 3 - cmake/FindRestbed.cmake | 1 - include/opendht/base64.h | 35 +++++++- include/opendht/dhtproxyserver.h | 45 ++++++++-- include/opendht/dhtrunner.h | 11 ++- include/opendht/value.h | 13 ++- src/base64.cpp | 149 +++++++++++++++++-------------- src/dhtproxyserver.cpp | 30 ++++++- src/dhtrunner.cpp | 4 +- src/value.cpp | 7 +- tools/dhtchat.cpp | 2 - tools/dhtnode.cpp | 26 +++++- 12 files changed, 230 insertions(+), 96 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c94b818f..333b1a6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,9 +84,6 @@ endif () if (Nettle_INCLUDE_DIRS) include_directories (SYSTEM "${Nettle_INCLUDE_DIRS}") endif () -if (Restbed_INCLUDE_DIRS) - include_directories (SYSTEM "${Restbed_INCLUDE_DIRS}") -endif () if (Jsoncpp_INCLUDE_DIRS) include_directories (SYSTEM "${Jsoncpp_INCLUDE_DIRS}") endif () diff --git a/cmake/FindRestbed.cmake b/cmake/FindRestbed.cmake index 440e5596..05f1e814 100644 --- a/cmake/FindRestbed.cmake +++ b/cmake/FindRestbed.cmake @@ -14,5 +14,4 @@ endif() if (RESTBED_INCLUDE) set(RESTBED_FOUND TRUE) set(Restbed_LIBRARIES restbed) - set(Restbed_INCLUDE_DIRS ${RESTBED_INCLUDE}/restbed) endif() diff --git a/include/opendht/base64.h b/include/opendht/base64.h index 8f4d9ebb..294762a8 100644 --- a/include/opendht/base64.h +++ b/include/opendht/base64.h @@ -1,8 +1,39 @@ +/* + * Copyright (C) 2004-2017 Savoir-faire Linux Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + #if OPENDHT_PROXY_SERVER #pragma once #include +#include + +/** + * Encode a string into base64 + * @param begin + * @param end + * @return the encoded vector in a string + */ +std::string base64_encode(const std::vector& str); +/** + * Decode a base64 encoded string + * @param str to decode + * @return the decoded string + */ +std::string base64_decode(std::string const& str); -std::string base64_encode(unsigned char const* , unsigned int len); -std::string base64_decode(std::string const& s); #endif // OPENDHT_PROXY_SERVER diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dhtproxyserver.h index 17f13f9a..52d9f0e1 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dhtproxyserver.h @@ -1,4 +1,20 @@ -// TODO gpl +/* + * Copyright (C) 2016 Savoir-faire Linux Inc. + * Author : Sébastien Blin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ #if OPENDHT_PROXY_SERVER @@ -14,9 +30,19 @@ namespace dht { class DhtRunner; +/** + * Describes the REST API + */ class OPENDHT_PUBLIC DhtProxyServer { public: + /** + * Start the Http server for OpenDHT + * @param dht the DhtRunner linked to this proxy server + * @param port to listen + * @note if the server fails to start (if port is already used or reserved), + * it will fails silently + */ DhtProxyServer(DhtRunner* dht, unsigned int port = 8000); virtual ~DhtProxyServer(); @@ -25,13 +51,16 @@ public: DhtProxyServer& operator=(const DhtProxyServer& other) = default; DhtProxyServer& operator=(DhtProxyServer&& other) = default; + /** + * Stop the DhtProxyServer + */ void stop(); private: /** * Return the PublicKey id, the node id and node stats * Method: GET "/" - * Result: HTTP 200, body: {"id":"xxxx", "node_id":"xxxx", "ipv4":"xxxxxx", "ipv6": "xxxxx"} + * Result: HTTP 200, body: Value in JSON format (one part = one value) * On error: HTTP 404, body: {"err":"xxxx"} */ void getNodeInfo(const std::shared_ptr& session) const; @@ -40,8 +69,8 @@ private: * Return Values of a InfoHash * Method: GET "/{InfoHash: .*}" * Return: Multiple JSON object in parts. For 2 values, you will have 3 parts: - * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) - * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) + * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) * {"ok": 1} * * On error: HTTP 404, body: {"err":"xxxx"} @@ -52,8 +81,8 @@ private: * Listen incoming Values of a InfoHash. * Method: LISTEN "/{InfoHash: .*}" * Return: Multiple JSON object in parts. For 2 values, you will have 2 parts: - * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) - * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) + * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) * * On error: HTTP 404, body: {"err":"xxxx"} */ @@ -95,8 +124,8 @@ private: * Return Values of a InfoHash filtered by a value id * Method: GET "/{InfoHash: .*}/{ValueId: .*}" * Return: Multiple JSON object in parts. For 2 values, you will have 3 parts: - * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) - * {"data":"xxxx" (...) "type":3} (HTTP/1.1 200 OK Content-Type: application/json) + * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) + * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) * {"ok": 1} * * On error: HTTP 404, body: {"err":"xxxx"} diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index 8e6cfd6b..acaac185 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -367,8 +367,15 @@ public: void join(); #if OPENDHT_PROXY_SERVER - void startProxyInterface(unsigned int port=8000); - void stopProxyInterface(); + /** + * Start the proxy server + * @param port used for the server + */ + void startProxyServer(unsigned int port=8000); + /** + * Stop the proxy server + */ + void stopProxyServer(); #endif //OPENDHT_PROXY_SERVER private: diff --git a/include/opendht/value.h b/include/opendht/value.h index 4119d7ab..e27e0f12 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -352,8 +352,11 @@ struct OPENDHT_PUBLIC Value : id(id), type(t), data(dat_ptr, dat_ptr+dat_len) {} #if OPENDHT_PROXY_SERVER + /** + * Build a value from a json object + * @param json + */ Value(Json::Value& json) { - try { if (json.isMember("id")) id = ValueType::Id(json["id"].asInt()); @@ -463,6 +466,14 @@ struct OPENDHT_PUBLIC Value } #if OPENDHT_PROXY_SERVER + /** + * Build a json object from a value + * Example: + * { + * "data":"base64ofdata", + * id":"0", "seq":0,"type":3 + * } + */ Json::Value toJson() const; #endif //OPENDHT_PROXY_SERVER diff --git a/src/base64.cpp b/src/base64.cpp index 4068c003..2732f40d 100644 --- a/src/base64.cpp +++ b/src/base64.cpp @@ -1,97 +1,114 @@ +/* + * Copyright (C) 2004-2017 Savoir-faire Linux Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ #if OPENDHT_PROXY_SERVER #include "base64.h" static const std::string base64_chars = - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/"; + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; static inline bool is_base64(unsigned char c) { - return (isalnum(c) || (c == '+') || (c == '/')); + return (isalnum(c) || (c == '+') || (c == '/')); } -std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { - std::string ret; - int i = 0; - int j = 0; - unsigned char char_array_3[3]; - unsigned char char_array_4[4]; +std::string base64_encode(const std::vector& str) { + unsigned char const* bytes_to_encode = str.data(); + unsigned int in_len = str.size(); + std::string ret; + int i = 0; + int j = 0; + unsigned char char_array_3[3]; + unsigned char char_array_4[4]; - while (in_len--) { - char_array_3[i++] = *(bytes_to_encode++); - if (i == 3) { - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; + while (in_len--) { + char_array_3[i++] = *(bytes_to_encode++); + if (i == 3) { + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; - for(i = 0; (i <4) ; i++) - ret += base64_chars[char_array_4[i]]; - i = 0; + for(i = 0; (i <4) ; i++) + ret += base64_chars[char_array_4[i]]; + i = 0; + } } - } - if (i) - { - for(j = i; j < 3; j++) - char_array_3[j] = '\0'; + if (i) + { + for(j = i; j < 3; j++) + char_array_3[j] = '\0'; - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; - for (j = 0; (j < i + 1); j++) - ret += base64_chars[char_array_4[j]]; + for (j = 0; (j < i + 1); j++) + ret += base64_chars[char_array_4[j]]; - while((i++ < 3)) - ret += '='; - - } - - return ret; + while((i++ < 3)) + ret += '='; + } + return ret; } std::string base64_decode(std::string const& encoded_string) { - size_t in_len = encoded_string.size(); - size_t i = 0; - size_t j = 0; - int in_ = 0; - unsigned char char_array_4[4], char_array_3[3]; - std::string ret; + size_t in_len = encoded_string.size(); + size_t i = 0; + size_t j = 0; + int in_ = 0; + unsigned char char_array_4[4], char_array_3[3]; + std::string ret; - while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { - char_array_4[i++] = encoded_string[in_]; in_++; - if (i ==4) { - for (i = 0; i <4; i++) - char_array_4[i] = static_cast(base64_chars.find(char_array_4[i])); + while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { + char_array_4[i++] = encoded_string[in_]; in_++; + if (i ==4) { + for (i = 0; i <4; i++) + char_array_4[i] = static_cast(base64_chars.find(char_array_4[i])); - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - for (i = 0; (i < 3); i++) - ret += char_array_3[i]; - i = 0; + for (i = 0; (i < 3); i++) + ret += char_array_3[i]; + i = 0; + } } - } - if (i) { - for (j = i; j <4; j++) - char_array_4[j] = 0; + if (i) { + for (j = i; j <4; j++) + char_array_4[j] = 0; - for (j = 0; j <4; j++) - char_array_4[j] = static_cast(base64_chars.find(char_array_4[j])); + for (j = 0; j <4; j++) + char_array_4[j] = static_cast(base64_chars.find(char_array_4[j])); - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; - } + for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; + } - return ret; + return ret; } #endif // OPENDHT_PROXY_SERVER diff --git a/src/dhtproxyserver.cpp b/src/dhtproxyserver.cpp index de35c8cf..8557f874 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dhtproxyserver.cpp @@ -1,4 +1,20 @@ -// TODO GPL +/* + * Copyright (C) 2016 Savoir-faire Linux Inc. + * Author : Sébastien Blin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ #if OPENDHT_PROXY_SERVER #include "dhtproxyserver.h" @@ -117,6 +133,7 @@ DhtProxyServer::getNodeInfo(const std::shared_ptr& session) co session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) { + (void)b; if (dht_) { Json::Value result; result["id"] = dht_->getId().toString(); @@ -142,6 +159,7 @@ DhtProxyServer::get(const std::shared_ptr& session) const session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) { + (void)b; if (dht_) { InfoHash infoHash(hash); if (infoHash == INVALID_ID) { @@ -149,10 +167,12 @@ DhtProxyServer::get(const std::shared_ptr& session) const } Json::FastWriter writer; dht_->get(infoHash, [s, &writer](std::shared_ptr value) { + // Send values as soon as we get them Json::Value result; s->yield(restbed::OK, writer.write(value->toJson())); return true; }, [s, &writer](bool ok) { + // Communication is finished auto response = std::to_string(ok); s->close(restbed::OK, "{\"ok\": " + response + "}"); }); @@ -177,15 +197,17 @@ DhtProxyServer::listen(const std::shared_ptr& session) const session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) { + (void)b; if (dht_) { Json::FastWriter writer; token = dht_->listen(infoHash, [s, &writer](std::shared_ptr value) { - Json::Value result; + // Send values as soon as we get them if (!s->is_closed()) s->yield(restbed::OK, writer.write(value->toJson())); return !s->is_closed(); }).get(); // Handle client deconnection + // NOTE: for now, there is no handler, so we test the session in a thread listenThreads_.emplace_back(std::shared_ptr( new std::thread([&]() { while(!s->is_closed()) @@ -205,7 +227,7 @@ DhtProxyServer::listen(const std::shared_ptr& session) const void DhtProxyServer::put(const std::shared_ptr& session) const { - // TODO test with encrypted and signed value to send + // TODO test with the proxy client static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); @@ -228,6 +250,7 @@ DhtProxyServer::put(const std::shared_ptr& session) const std::string strJson(buf.begin(), buf.end()); bool parsingSuccessful = reader.parse(strJson.c_str(), root); if (parsingSuccessful) { + // Build the Value from json auto value = std::make_shared(root); auto response = value->toString(); @@ -347,6 +370,7 @@ DhtProxyServer::getFiltered(const std::shared_ptr& session) co session->fetch(content_length, [&](const std::shared_ptr s, const restbed::Bytes& b) { + (void)b; if (dht_) { InfoHash infoHash(hash); if (infoHash == INVALID_ID) { diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index c33ba231..8988c3e8 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -802,14 +802,14 @@ DhtRunner::findCertificate(InfoHash hash, std::function(new DhtProxyServer(this, port)); } void -DhtRunner::stopProxyInterface() +DhtRunner::stopProxyServer() { if (proxy_server_) proxy_server_.reset(nullptr); diff --git a/src/value.cpp b/src/value.cpp index 1565ce8d..a4e02bc4 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -176,14 +176,13 @@ Value::msgpack_unpack_body(const msgpack::object& o) Json::Value Value::toJson() const { - // TODO check Json::Value val; val["id"] = std::to_string(id); if (isEncrypted()) { - val["cypher"] = base64_encode(cypher.data(), cypher.size()); + val["cypher"] = base64_encode(cypher); } else { if (isSigned()) - val["sig"] = base64_encode(signature.data(), signature.size()); + val["sig"] = base64_encode(signature); bool has_owner = owner && *owner; if (has_owner) { // isSigned val["seq"] = seq; @@ -192,7 +191,7 @@ Value::toJson() const val["to"] = recipient.toString(); } val["type"] = type; - val["data"] = base64_encode(data.data(), data.size()); + val["data"] = base64_encode(data); if (not user_type.empty()) val["utype"] = user_type; } diff --git a/tools/dhtchat.cpp b/tools/dhtchat.cpp index b64b4c95..8e48723f 100644 --- a/tools/dhtchat.cpp +++ b/tools/dhtchat.cpp @@ -67,8 +67,6 @@ main(int argc, char **argv) if (not params.bootstrap.first.empty()) dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str()); - dht.startProxyInterface(); - print_node_info(dht, params); std::cout << " type 'c {hash}' to join a channel" << std::endl << std::endl; diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 50255e50..cb4dc125 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -55,16 +55,23 @@ void print_help() { << " ll Print basic information and stats about the current node." << std::endl << " ls [key] Print basic information about current search(es)." << std::endl << " ld [key] Print basic information about currenty stored values on this node (or key)." << std::endl - << " lr Print the full current routing table of this node" << std::endl; + << " lr Print the full current routing table of this node." << std::endl; + +#if OPENDHT_PROXY_SERVER + std::cout << std::endl << "Operations with the proxy:" << std::endl + << " psr [port] Start the proxy interface on port." << std::endl + << " pst Stop the proxy interface." << std::endl; +#endif //OPENDHT_PROXY_SERVER std::cout << std::endl << "Operations on the DHT:" << std::endl - << " b Ping potential node at given IP address/port." << std::endl + << " b Ping potential node at given IP address/port." << std::endl << " g Get values at ." << std::endl << " l Listen for value changes at ." << std::endl << " p Put string value at ." << std::endl << " pp Put string value at (persistent version)." << std::endl << " s Put string value at , signed with our generated private key." << std::endl << " e Put string value at , encrypted for with its public key (if found)." << std::endl; + std::cout << std::endl << "Indexation operations on the DHT:" << std::endl << " il [exact match] Lookup the index named with the key ." << std::endl << " Set [exact match] to 'false' for inexact match lookup." << std::endl @@ -162,6 +169,21 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) dht->setLogFilter(filter); continue; } +#if OPENDHT_PROXY_SERVER + else if (op == "psr") { + iss >> idstr; + try { + unsigned int port = std::stoi(idstr); + dht->startProxyServer(port); + } catch (...) { + dht->startProxyServer(); + } + continue; + } else if (op == "pst") { + dht->stopProxyServer(); + continue; + } +#endif //OPENDHT_PROXY_SERVER if (op.empty()) continue; From 61a148e83f53596f8217879b0dd456eac0d9990e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 1 Nov 2017 14:08:42 -0400 Subject: [PATCH 077/156] update README.md --- README.md | 8 ++++++-- tools/dhtnode.cpp | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0b6d58bb..885afaff 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

- OpenDHT + OpenDHT

A lightweight C++11 Distributed Hash Table implementation originally based on https://github.com/jech/dht by Juliusz Chroboczek. @@ -18,7 +18,7 @@ See the wiki: #### How-to build and install -Build instructions : +Build instructions: #### How-to build a simple client app ```bash @@ -94,6 +94,10 @@ for r in results: - Build tested with GCC 4.8+ (GNU/Linux, Android, Windows with MinGW), Clang/LLVM (Linux, macOS). - Build tested with Microsoft Visual Studio 2015 +### Optional dependencies +- restbed 4.0+, used for the REST API. +- jsoncpp 1.7.4-3+, used for the REST API. + ## Contact IRC: join us on Freenode at [`#opendht`](https://webchat.freenode.net/?channels=%23opendht). diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index cb4dc125..31f845a0 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -59,8 +59,8 @@ void print_help() { #if OPENDHT_PROXY_SERVER std::cout << std::endl << "Operations with the proxy:" << std::endl - << " psr [port] Start the proxy interface on port." << std::endl - << " pst Stop the proxy interface." << std::endl; + << " pst [port] Start the proxy interface on port." << std::endl + << " psp Stop the proxy interface." << std::endl; #endif //OPENDHT_PROXY_SERVER std::cout << std::endl << "Operations on the DHT:" << std::endl @@ -170,7 +170,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) continue; } #if OPENDHT_PROXY_SERVER - else if (op == "psr") { + else if (op == "pst") { iss >> idstr; try { unsigned int port = std::stoi(idstr); @@ -179,7 +179,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) dht->startProxyServer(); } continue; - } else if (op == "pst") { + } else if (op == "psp") { dht->stopProxyServer(); continue; } From 834d62e6e99b7d162a81f706bc94df393ef2f570 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Wed, 1 Nov 2017 16:28:17 -0400 Subject: [PATCH 078/156] InfoHash: use bool operator where relevent --- include/opendht/log_enable.h | 2 +- tools/dhtchat.cpp | 4 +--- tools/dhtnode.cpp | 14 ++++++-------- tools/tools_common.h | 6 ++---- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/include/opendht/log_enable.h b/include/opendht/log_enable.h index b55f5556..7310de54 100644 --- a/include/opendht/log_enable.h +++ b/include/opendht/log_enable.h @@ -75,7 +75,7 @@ struct Logger { LogMethod ERR = NOLOG; void setFilter(const InfoHash& f) { filter_ = f; - filterEnable_ = filter_ != InfoHash{}; + filterEnable_ = static_cast(filter_); } inline void log0(const LogMethod& logger, char const* format, va_list args) const { #if OPENDHT_LOG diff --git a/tools/dhtchat.cpp b/tools/dhtchat.cpp index 8e48723f..3bd89f1c 100644 --- a/tools/dhtchat.cpp +++ b/tools/dhtchat.cpp @@ -88,8 +88,6 @@ main(int argc, char **argv) if (line.empty()) continue; - static constexpr dht::InfoHash INVALID_ID {}; - std::istringstream iss(line); std::string op, idstr; iss >> op; @@ -99,7 +97,7 @@ main(int argc, char **argv) else if (op == "c") { iss >> idstr; room = InfoHash(idstr); - if (room == INVALID_ID) { + if (not room) { room = InfoHash::get(idstr); std::cout << "Joining h(" << idstr << ") = " << room << std::endl; } diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 96cb394d..a55d3d20 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -116,20 +116,18 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) } else if (op == "ld") { iss >> idstr; InfoHash filter(idstr); - if (filter == InfoHash{}) - std::cout << dht->getStorageLog() << std::endl; - else + if (filter) std::cout << dht->getStorageLog(filter) << std::endl; + else + std::cout << dht->getStorageLog() << std::endl; continue; } else if (op == "ls") { iss >> idstr; InfoHash filter(idstr); - if (filter == InfoHash{}) { - std::cout << "Searches:" << std::endl; - std::cout << dht->getSearchesLog() << std::endl; - } else { + if (filter) std::cout << dht->getSearchLog(filter) << std::endl; - } + else + std::cout << dht->getSearchesLog() << std::endl; continue; } else if (op == "la") { std::cout << "Reported public addresses:" << std::endl; diff --git a/tools/tools_common.h b/tools/tools_common.h index 933cc7e2..2e4d72b0 100644 --- a/tools/tools_common.h +++ b/tools/tools_common.h @@ -98,10 +98,8 @@ dht::indexation::Pht::Key createPhtKey(std::map pht_ke return pht_key; } -bool isInfoHash(dht::InfoHash& h) { - static constexpr dht::InfoHash INVALID_ID {}; - - if (h == INVALID_ID) { +bool isInfoHash(const dht::InfoHash& h) { + if (not h) { std::cout << "Syntax error: invalid InfoHash." << std::endl; return false; } From 9abc88f378927c8d30d22b4d760afeffe56141c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 1 Nov 2017 17:03:33 -0400 Subject: [PATCH 079/156] add --enable-proxy-server option for autotools --- configure.ac | 20 ++++++++++++++++++++ src/Makefile.am | 5 +++++ tools/Makefile.am | 6 ++++++ 3 files changed, 31 insertions(+) diff --git a/configure.ac b/configure.ac index 504de170..6689c437 100644 --- a/configure.ac +++ b/configure.ac @@ -128,6 +128,26 @@ AM_COND_IF([ENABLE_TOOLS], [ ]) ]) +AC_ARG_ENABLE([proxy_server], AS_HELP_STRING([--enable-proxy-server], [Enable proxy server ability]), proxy_server=yes, proxy_server=no) +AM_CONDITIONAL(ENABLE_PROXY_SERVER, test x$proxy_server == xyes) +AM_COND_IF([ENABLE_PROXY_SERVER], [ + AC_CHECK_LIB(restbed, exit,, AC_MSG_ERROR([Missing restbed files])) + PKG_CHECK_MODULES([Jsoncpp], [jsoncpp >= 1.7.4]) + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=true -ljsoncpp -lrestbed" + + AC_ARG_ENABLE([proxy_server_optionals], AS_HELP_STRING([--enable-proxy-server-optionals], + [Enable proxy server ability]), proxy_server_optionals=yes, proxy_server_optionals=no) + AM_CONDITIONAL(ENABLE_PROXY_SERVER_OPTIONALS, test x$proxy_server_optionals == xyes) + AM_COND_IF([ENABLE_PROXY_SERVER_OPTIONALS], [ + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_OPTIONAL=true" + ], [ + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_OPTIONAL=false" + ]) +], [ + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=false" +]) + + AC_CONFIG_FILES([doc/Doxyfile doc/Makefile]) dnl Configure setup.py if we build the python module diff --git a/src/Makefile.am b/src/Makefile.am index f4614795..e9dd2e9f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -54,6 +54,11 @@ nobase_include_HEADERS = \ ../include/opendht/rng.h \ ../include/opendht/indexation/pht.h +if ENABLE_PROXY_SERVER +libopendht_la_SOURCES += base64.cpp dhtproxyserver.cpp +nobase_include_HEADERS += ../include/opendht/base64.h ../include/opendht/dhtproxyserver.h +endif + clean-local: rm -rf libargon2.la diff --git a/tools/Makefile.am b/tools/Makefile.am index 610fd76f..821ddce1 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -11,3 +11,9 @@ dhtchat_LDFLAGS = -lopendht -lreadline -L@top_builddir@/src/.libs @Argon2_LDFLAG dhtscanner_SOURCES = dhtscanner.cpp dhtscanner_LDFLAGS = -lopendht -lreadline -L@top_builddir@/src/.libs @Argon2_LDFLAGS@ @GnuTLS_LIBS@ + +if ENABLE_PROXY_SERVER +dhtnode_LDFLAGS += -lrestbed -ljsoncpp +dhtchat_LDFLAGS += -lrestbed -ljsoncpp +dhtscanner_LDFLAGS += -lrestbed -ljsoncpp +endif From 87dc080bbfe945e6e505d77f6a14fe4f448191fa Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Thu, 2 Nov 2017 14:25:46 -0400 Subject: [PATCH 080/156] crypto: add raw hash function --- include/opendht/crypto.h | 2 ++ src/crypto.cpp | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/include/opendht/crypto.h b/include/opendht/crypto.h index 71648288..a840576b 100644 --- a/include/opendht/crypto.h +++ b/include/opendht/crypto.h @@ -601,6 +601,8 @@ OPENDHT_PUBLIC Identity generateEcIdentity(const std::string& name = "dhtnode", */ OPENDHT_PUBLIC Blob hash(const Blob& data, size_t hash_length = 512/8); +OPENDHT_PUBLIC void hash(const uint8_t* data, size_t data_length, uint8_t* hash, size_t hash_length); + /** * Generates an encryption key from a text password, * making the key longer to bruteforce. diff --git a/src/crypto.cpp b/src/crypto.cpp index 6c029d56..8e7d2a2c 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -84,7 +84,7 @@ static constexpr size_t PASSWORD_SALT_LENGTH {16}; constexpr gnutls_digest_algorithm_t gnutlsHashAlgo(size_t min_res) { return (min_res > 256/8) ? GNUTLS_DIG_SHA512 : ( - (min_res > 128/8) ? GNUTLS_DIG_SHA256 : ( + (min_res > 160/8) ? GNUTLS_DIG_SHA256 : ( GNUTLS_DIG_SHA1)); } @@ -212,12 +212,21 @@ Blob hash(const Blob& data, size_t hash_len) Blob res; res.resize(res_size); const gnutls_datum_t gdat {(uint8_t*)data.data(), (unsigned)data.size()}; - if (gnutls_fingerprint(algo, &gdat, res.data(), &res_size)) - throw CryptoException("Can't compute hash !"); + if (auto err = gnutls_fingerprint(algo, &gdat, res.data(), &res_size)) + throw CryptoException(std::string("Can't compute hash: ") + gnutls_strerror(err)); res.resize(std::min(hash_len, res_size)); return res; } +void hash(const uint8_t* data, size_t data_length, uint8_t* hash, size_t hash_length) +{ + auto algo = gnutlsHashAlgo(hash_length); + size_t res_size = hash_length; + const gnutls_datum_t gdat {(uint8_t*)data, (unsigned)data_length}; + if (auto err = gnutls_fingerprint(algo, &gdat, hash, &res_size)) + throw CryptoException(std::string("Can't compute hash: ") + gnutls_strerror(err)); +} + PrivateKey::PrivateKey() {} From 0e865976f972d4668c5468164be851d0d4c0a374 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Thu, 2 Nov 2017 14:28:09 -0400 Subject: [PATCH 081/156] InfoHash: unify with Hash --- include/opendht/infohash.h | 469 ++++++++++++++++++++----------------- src/infohash.cpp | 97 +------- src/routing_table.cpp | 2 +- 3 files changed, 253 insertions(+), 315 deletions(-) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index 57671186..aebb0cfe 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -19,6 +19,7 @@ #pragma once #include "def.h" +#include "rng.h" #include @@ -44,73 +45,244 @@ typedef uint16_t in_port_t; #include #include - -// bytes -#define HASH_LEN 20u - namespace dht { using byte = uint8_t; +namespace crypto { + OPENDHT_PUBLIC void hash(const uint8_t* data, size_t data_length, uint8_t* hash, size_t hash_length); +} + +/** + * Represents an InfoHash. + * An InfoHash is a byte array of HASH_LEN bytes. + * InfoHashes identify nodes and values in the Dht. + */ template -class OPENDHT_PUBLIC Hash -{ +class OPENDHT_PUBLIC Hash { public: - Hash() { + using T = std::array; + typedef typename T::iterator iterator; + typedef typename T::const_iterator const_iterator; + + Hash () { data_.fill(0); } + Hash (const uint8_t* h, size_t data_len) { + if (data_len < N) + data_.fill(0); + else + std::copy_n(h, N, data_.begin()); + } + /** + * Constructor from an hexadecimal string (without "0x"). + * hex must be at least 2.HASH_LEN characters long. + * If too long, only the first 2.HASH_LEN characters are read. + */ + explicit Hash(const std::string& hex) { + if (hex.size() < 2*N) { + data_.fill(0); + return; + } + const auto p = (const char*)hex.data(); + for (size_t i = 0; i < N; i++) { + unsigned res = 0; + sscanf(p + 2*i, "%02x", &res); + data_[i] = res; + } + } + Hash(const msgpack::object& o) { msgpack_unpack(o); } - byte* data() { - return data_.data(); + + size_t size() const { return data_.size(); } + const uint8_t* data() const { return data_.data(); } + uint8_t* data() { return data_.data(); } + iterator begin() { return data_.begin(); } + const_iterator cbegin() const { return data_.cbegin(); } + iterator end() { return data_.end(); } + const_iterator cend() const { return data_.cend(); } + + bool operator==(const Hash& h) const { + auto a = reinterpret_cast(data_.data()); + auto b = reinterpret_cast(h.data_.data()); + constexpr unsigned n = N / sizeof(uint32_t); + for (unsigned i=0; i < n; i++) + if (a[i] != b[i]) + return false; + return true; } - const byte* data() const { - return data_.data(); + bool operator!=(const Hash& h) const { return !(*this == h); } + + bool operator<(const Hash& o) const { + for(unsigned i = 0; i < N; i++) { + if(data_[i] != o.data_[i]) + return data_[i] < o.data_[i]; + } + return false; } - constexpr size_t size() const { return data_.size(); } - explicit operator bool() const { return std::any_of(data_.begin(), data_.end(), [](byte _b) { return _b != 0; }); } - bool operator==(Hash const& _c) const { return data_ == _c.data_; } - bool operator!=(Hash const& _c) const { return data_ != _c.data_; } - bool operator<(Hash const& _c) const { for (unsigned i = 0; i < N; ++i) if (data_[i] < _c.data_[i]) return true; else if (data_[i] > _c.data_[i]) return false; return false; } - bool operator>=(Hash const& _c) const { return !operator<(_c); } - bool operator<=(Hash const& _c) const { return operator==(_c) || operator<(_c); } - bool operator>(Hash const& _c) const { return !operator<=(_c); } - template - void msgpack_pack(Packer& pk) const + + explicit operator bool() const { + auto a = reinterpret_cast(data_.data()); + auto b = reinterpret_cast(data_.data() + N); + for (; a != b; a++) { + if (*a) + return true; + } + return false; + } + + uint8_t& operator[](size_t index) { return data_[index]; } + const uint8_t& operator[](size_t index) const { return data_[index]; } + + /** + * Find the lowest 1 bit in an id. + * Result will allways be lower than 8*N + */ + inline unsigned lowbit() const { + int i, j; + for(i = N-1; i >= 0; i--) + if(data_[i] != 0) + break; + if(i < 0) + return -1; + for(j = 7; j >= 0; j--) + if((data_[i] & (0x80 >> j)) != 0) + break; + return 8 * i + j; + } + + /** + * Forget about the ``XOR-metric''. An id is just a path from the + * root of the tree, so bits are numbered from the start. + */ + static inline int cmp(const Hash& id1, const Hash& id2) { + return std::memcmp(id1.data_.data(), id2.data_.data(), N); + } + + /** Find how many bits two ids have in common. */ + static inline unsigned + commonBits(const Hash& id1, const Hash& id2) { - pk.pack_bin(N); - pk.pack_bin_body((char*)data_.data(), N); + unsigned i, j; + uint8_t x; + for(i = 0; i < N; i++) { + if(id1.data_[i] != id2.data_[i]) + break; + } + + if(i == N) + return 8*N; + + x = id1.data_[i] ^ id2.data_[i]; + + j = 0; + while((x & 0x80) == 0) { + x <<= 1; + j++; + } + + return 8 * i + j; } - void msgpack_unpack(msgpack::object o) { - if (o.type != msgpack::type::BIN or o.via.bin.size != N) - throw msgpack::type_error(); - std::copy_n(o.via.bin.ptr, N, data_.data()); + + /** Determine whether id1 or id2 is closer to this */ + int + xorCmp(const Hash& id1, const Hash& id2) const + { + for(unsigned i = 0; i < N; i++) { + uint8_t xor1, xor2; + if(id1.data_[i] == id2.data_[i]) + continue; + xor1 = id1.data_[i] ^ data_[i]; + xor2 = id2.data_[i] ^ data_[i]; + if(xor1 < xor2) + return -1; + else + return 1; + } + return 0; } + bool + getBit(unsigned nbit) const + { + auto& num = *(data_.cbegin()+(nbit/8)); + unsigned bit = 7 - (nbit % 8); + return (num >> bit) & 1; + } + + void + setBit(unsigned nbit, bool b) + { + auto& num = data_[nbit/8]; + unsigned bit = 7 - (nbit % 8); + num ^= (-b ^ num) & (1 << bit); + } + + double toFloat() const { + using D = size_t; + double v = 0.; + for (size_t i = 0; i < std::min(N, sizeof(D)-1); i++) + v += *(data_.cbegin()+i) / (double)((D)1 << 8*(i+1)); + return v; + } + + static inline Hash get(const std::string& data) { + return get((const uint8_t*)data.data(), data.size()); + } + + static inline Hash get(const std::vector& data) { + return get(data.data(), data.size()); + } + + /** + * Computes the hash from a given data buffer of size data_len. + */ + static Hash get(const uint8_t* data, size_t data_len) + { + Hash ret; + crypto::hash(data, data_len, ret.data(), N); + return ret; + } + + static Hash getRandom(); + template OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const Hash& h); template OPENDHT_PUBLIC friend std::istream& operator>> (std::istream& s, Hash& h); - std::string toString() const { - std::stringstream ss; - ss << *this; - return ss.str(); + const char* to_c_str() const; + + std::string toString() const; + + template + void msgpack_pack(Packer& pk) const + { + pk.pack_bin(N); + pk.pack_bin_body((char*)data_.data(), N); } + void msgpack_unpack(msgpack::object o) { + if (o.type != msgpack::type::BIN or o.via.bin.size != N) + throw msgpack::type_error(); + std::copy_n(o.via.bin.ptr, N, data_.data()); + } private: - std::array data_; + T data_; }; +#define HASH_LEN 20u +using InfoHash = Hash; +using h256 = Hash<32>; +using PkId = h256; + template std::ostream& operator<< (std::ostream& s, const Hash& h) { - s << std::hex; - for (unsigned i=0; i> (std::istream& s, Hash& h) return s; } -using h256 = Hash<32>; -using PkId = h256; +template +Hash +Hash::getRandom() +{ + Hash h; + crypto::random_device rdev; + std::uniform_int_distribution rand_int; + auto a = reinterpret_cast(h.data()); + auto b = reinterpret_cast(h.data() + h.size()); + std::generate(a, b, std::bind(rand_int, std::ref(rdev))); + return h; +} -/** - * Represents an InfoHash. - * An InfoHash is a byte array of HASH_LEN bytes. - * InfoHashes identify nodes and values in the Dht. - */ -class OPENDHT_PUBLIC InfoHash final : public std::array { -public: - constexpr InfoHash() : std::array({{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}}) {} - constexpr InfoHash(const std::array& h) : std::array(h) {} - InfoHash(const uint8_t* h, size_t h_len) : std::array() { - if (h_len < HASH_LEN) - fill(0); - else - std::copy_n(h, HASH_LEN, begin()); - } - - /** - * Constructor from an hexadecimal string (without "0x"). - * hex must be at least 2.HASH_LEN characters long. - * If too long, only the first 2.HASH_LEN characters are read. - */ - explicit InfoHash(const std::string& hex); - - InfoHash(const msgpack::object& o) { - msgpack_unpack(o); - } - - /** - * Find the lowest 1 bit in an id. - * Result will allways be lower than 8*HASH_LEN - */ - inline unsigned lowbit() const { - int i, j; - for(i = HASH_LEN-1; i >= 0; i--) - if((*this)[i] != 0) - break; - if(i < 0) - return -1; - for(j = 7; j >= 0; j--) - if(((*this)[i] & (0x80 >> j)) != 0) - break; - return 8 * i + j; - } - - /** - * Forget about the ``XOR-metric''. An id is just a path from the - * root of the tree, so bits are numbered from the start. - */ - static inline int cmp(const InfoHash& id1, const InfoHash& id2) { - return std::memcmp(id1.data(), id2.data(), HASH_LEN); - } - - /** Find how many bits two ids have in common. */ - static inline unsigned - commonBits(const InfoHash& id1, const InfoHash& id2) - { - unsigned i, j; - uint8_t x; - for(i = 0; i < HASH_LEN; i++) { - if(id1[i] != id2[i]) - break; +struct HexMap : public std::array, 256> { + HexMap() { + for (size_t i=0; i> 4) & 0x0F]; + e[1] = hex_digits[i & 0x0F]; } - - if(i == HASH_LEN) - return 8*HASH_LEN; - - x = id1[i] ^ id2[i]; - - j = 0; - while((x & 0x80) == 0) { - x <<= 1; - j++; - } - - return 8 * i + j; } - - /** Determine whether id1 or id2 is closer to this */ - int - xorCmp(const InfoHash& id1, const InfoHash& id2) const - { - for(unsigned i = 0; i < HASH_LEN; i++) { - uint8_t xor1, xor2; - if(id1[i] == id2[i]) - continue; - xor1 = id1[i] ^ (*this)[i]; - xor2 = id2[i] ^ (*this)[i]; - if(xor1 < xor2) - return -1; - else - return 1; - } - return 0; - } - - bool - getBit(unsigned nbit) const - { - auto& num = *(cbegin()+(nbit/8)); - unsigned bit = 7 - (nbit % 8); - return (num >> bit) & 1; - } - - void - setBit(unsigned nbit, bool b) - { - auto& num = (*this)[nbit/8]; - unsigned bit = 7 - (nbit % 8); - num ^= (-b ^ num) & (1 << bit); - } - - double toFloat() const { - using D = size_t; - double v = 0.; - for (size_t i = 0; i < std::min(HASH_LEN, sizeof(D)-1); i++) - v += *(cbegin()+i) / (double)((D)1 << 8*(i+1)); - return v; - } - - bool operator==(const InfoHash& h) const { - auto a = reinterpret_cast(data()); - auto b = reinterpret_cast(h.data()); - for (size_t i=0; i<(size()/sizeof(uint32_t)); i++) - if (a[i] != b[i]) - return false; - return true; - } - - bool operator<(const InfoHash& o) const { - for(unsigned i = 0; i < HASH_LEN; i++) { - if((*this)[i] != o[i]) - return (*this)[i] < o[i]; - } - return false; - } - - explicit operator bool() const { - auto a = reinterpret_cast(data()); - auto b = reinterpret_cast(data() + size()); - for (; a != b; a++) { - if (*a) - return true; - } - return false; - } - - static inline InfoHash get(const std::string& data) { - return get((const uint8_t*)data.data(), data.size()); - } - - static inline InfoHash get(const std::vector& data) { - return get(data.data(), data.size()); - } - - /** - * Computes the hash from a given data buffer of size data_len. - */ - static InfoHash get(const uint8_t* data, size_t data_len); - - static InfoHash getRandom(); - - OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const InfoHash& h); - OPENDHT_PUBLIC friend std::istream& operator>> (std::istream& s, InfoHash& h); - - const char* to_c_str() const; - - std::string toString() const; - - template - void msgpack_pack(Packer& pk) const - { - pk.pack_bin(HASH_LEN); - pk.pack_bin_body((char*)data(), HASH_LEN); - } - - void msgpack_unpack(msgpack::object o) { - if (o.type != msgpack::type::BIN or o.via.bin.size != HASH_LEN) - throw msgpack::type_error(); - std::copy_n(o.via.bin.ptr, HASH_LEN, data()); - } - +private: + static constexpr const char* hex_digits = "0123456789abcdef"; }; -static constexpr const InfoHash zeroes {}; +OPENDHT_PUBLIC extern const HexMap hex_map; + +template +const char* +Hash::to_c_str() const +{ + thread_local std::array buf; + for (size_t i=0; i +std::string +Hash::toString() const +{ + return std::string(to_c_str(), N*2); +} + +const InfoHash zeroes {}; struct OPENDHT_PUBLIC NodeExport { InfoHash id; diff --git a/src/infohash.cpp b/src/infohash.cpp index 3c4e0eaf..9e9f9a61 100644 --- a/src/infohash.cpp +++ b/src/infohash.cpp @@ -17,11 +17,6 @@ */ #include "infohash.h" -#include "rng.h" - -extern "C" { -#include -} #include #include @@ -29,97 +24,7 @@ extern "C" { namespace dht { -InfoHash::InfoHash(const std::string& hex) { - if (hex.size() < 2*HASH_LEN) { - fill(0); - return; - } - const auto p = (const char*)hex.data(); - for (size_t i = 0; i < HASH_LEN; i++) { - unsigned res = 0; - sscanf(p + 2*i, "%02x", &res); - (*this)[i] = res; - } -} - -InfoHash -InfoHash::get(const uint8_t* data, size_t data_len) -{ - InfoHash h; - size_t s = h.size(); - const gnutls_datum_t gnudata = {(uint8_t*)data, (unsigned)data_len}; - const gnutls_digest_algorithm_t algo = (HASH_LEN == 64) ? GNUTLS_DIG_SHA512 : ( - (HASH_LEN == 32) ? GNUTLS_DIG_SHA256 : ( - (HASH_LEN == 20) ? GNUTLS_DIG_SHA1 : - GNUTLS_DIG_NULL )); - static_assert(algo != GNUTLS_DIG_NULL, "Can't find hash function to use."); - int rc = gnutls_fingerprint(algo, &gnudata, h.data(), &s); - if (rc == 0 && s == HASH_LEN) - return h; - throw std::runtime_error(std::string("Error hashing: ") + gnutls_strerror(rc)); -} - -InfoHash -InfoHash::getRandom() -{ - InfoHash h; - crypto::random_device rdev; - std::uniform_int_distribution rand_int; - auto a = reinterpret_cast(h.data()); - auto b = reinterpret_cast(h.data() + h.size()); - std::generate(a, b, std::bind(rand_int, std::ref(rdev))); - return h; -} - -struct HexMap : public std::array, 256> { - HexMap() { - for (size_t i=0; i> 4) & 0x0F]; - e[1] = hex_digits[i & 0x0F]; - } - } -private: - static constexpr const char* hex_digits = "0123456789abcdef"; -}; - -const char* -InfoHash::to_c_str() const -{ - static const HexMap map; - thread_local std::array buf; - for (size_t i=0; i> (std::istream& s, InfoHash& h) -{ - std::array dat; - s.exceptions(std::istream::eofbit | std::istream::failbit); - s.read(&(*dat.begin()), dat.size()); - for (size_t i = 0; i < HASH_LEN; i++) { - unsigned res = 0; - sscanf(dat.data() + 2*i, "%02x", &res); - h[i] = res; - } - return s; -} +const HexMap hex_map = {}; void NodeExport::msgpack_unpack(msgpack::object o) diff --git a/src/routing_table.cpp b/src/routing_table.cpp index b0a95e9d..e9a5bb15 100644 --- a/src/routing_table.cpp +++ b/src/routing_table.cpp @@ -76,7 +76,7 @@ RoutingTable::randomId(const RoutingTable::const_iterator& it) const int b = bit/8; InfoHash id_return; - std::copy_n(it->first.begin(), b, id_return.begin()); + std::copy_n(it->first.cbegin(), b, id_return.begin()); id_return[b] = it->first[b] & (0xFF00 >> (bit % 8)); id_return[b] |= rand_byte(rd) >> (bit % 8); for (unsigned i = b + 1; i < HASH_LEN; i++) From e9eb97ea9ab62a6095c0b0faf88bd1ef7a63b8c3 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Thu, 2 Nov 2017 17:12:06 -0400 Subject: [PATCH 082/156] InfoHash: improve hex parsing performance --- include/opendht/infohash.h | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index aebb0cfe..d7ae4070 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -79,18 +79,7 @@ public: * hex must be at least 2.HASH_LEN characters long. * If too long, only the first 2.HASH_LEN characters are read. */ - explicit Hash(const std::string& hex) { - if (hex.size() < 2*N) { - data_.fill(0); - return; - } - const auto p = (const char*)hex.data(); - for (size_t i = 0; i < N; i++) { - unsigned res = 0; - sscanf(p + 2*i, "%02x", &res); - data_[i] = res; - } - } + explicit Hash(const std::string& hex); Hash(const msgpack::object& o) { msgpack_unpack(o); @@ -272,6 +261,7 @@ public: } private: T data_; + void fromString(const char*); }; #define HASH_LEN 20u @@ -292,12 +282,29 @@ std::istream& operator>> (std::istream& s, Hash& h) std::array dat; s.exceptions(std::istream::eofbit | std::istream::failbit); s.read(&(*dat.begin()), dat.size()); - for (size_t i = 0; i < h.size(); i++) { - unsigned res = 0; - sscanf(dat.data() + 2*i, "%02x", &res); - h[i] = res; + fromString(dat.data()); +} + +template +Hash::Hash(const std::string& hex) { + if (hex.size() < 2*N) { + data_.fill(0); + return; + } + fromString(hex.c_str()); +} + +template +void +Hash::fromString(const char* in) { + auto hex2bin = [](char c) -> uint8_t { + if (c >= 'a') return 10 + c - 'a'; + else if (c >= 'A') return 10 + c - 'A'; + else return c - '0'; + }; + for (size_t i=0; i From 88b4233bdab953c084c4d1b1d9b8f1b9ae69f72a Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Fri, 3 Nov 2017 13:55:33 -0400 Subject: [PATCH 083/156] python: update InfoHash bool operator --- python/opendht.pyx | 2 +- python/opendht_cpp.pxd | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/opendht.pyx b/python/opendht.pyx index 87cb1c19..1c25ce9a 100644 --- a/python/opendht.pyx +++ b/python/opendht.pyx @@ -94,7 +94,7 @@ cdef class InfoHash(_WithID): def __cinit__(self, bytes str=b''): self._infohash = cpp.InfoHash(str) def __bool__(InfoHash self): - return not (self._infohash == cpp.InfoHash()) + return self._infohash def __richcmp__(InfoHash self, InfoHash other, int op): if op == 0: return self._infohash < other._infohash diff --git a/python/opendht_cpp.pxd b/python/opendht_cpp.pxd index 1ad79b56..3b889971 100644 --- a/python/opendht_cpp.pxd +++ b/python/opendht_cpp.pxd @@ -65,8 +65,9 @@ cdef extern from "opendht/infohash.h" namespace "dht": InfoHash get(string s) @staticmethod InfoHash getRandom() - bool operator==(InfoHash) - bool operator<(InfoHash) + bool operator==(InfoHash) const + bool operator<(InfoHash) const + bool operator bool() const cdef extern from "opendht/sockaddr.h" namespace "dht": cdef cppclass SockAddr: @@ -259,4 +260,3 @@ cdef extern from "opendht/indexation/pht.h" namespace "dht::indexation": void insert(IndexKey k, IndexValue v, DoneCallbackSimple cb) @staticmethod LookupCallback bindLookupCb(LookupCallbackRaw cb, void *user_data) - From bf4e838e206b99649772e9cfdedc34553f175c77 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Fri, 3 Nov 2017 13:40:13 -0400 Subject: [PATCH 084/156] InfoHash: return stream in operator>> --- include/opendht/infohash.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index d7ae4070..dcefcd0c 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -283,6 +283,7 @@ std::istream& operator>> (std::istream& s, Hash& h) s.exceptions(std::istream::eofbit | std::istream::failbit); s.read(&(*dat.begin()), dat.size()); fromString(dat.data()); + return s; } template From 92320cef59068bd1f64f485f73932f370b9baaac Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Fri, 3 Nov 2017 14:00:31 -0400 Subject: [PATCH 085/156] InfoHash: throw in case of invalid input string --- include/opendht/infohash.h | 16 +++++++++------- python/opendht.pyx | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index dcefcd0c..5acc96f8 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -288,20 +288,22 @@ std::istream& operator>> (std::istream& s, Hash& h) template Hash::Hash(const std::string& hex) { - if (hex.size() < 2*N) { + if (hex.empty()) data_.fill(0); - return; - } - fromString(hex.c_str()); + else if (hex.size() < 2*N) + throw std::invalid_argument("string not long enough"); + else + fromString(hex.c_str()); } template void Hash::fromString(const char* in) { auto hex2bin = [](char c) -> uint8_t { - if (c >= 'a') return 10 + c - 'a'; - else if (c >= 'A') return 10 + c - 'A'; - else return c - '0'; + if (c >= 'a' and c <= 'f') return 10 + c - 'a'; + else if (c >= 'A' and c <= 'F') return 10 + c - 'A'; + else if (c >= '0' and c <= '9') return c - '0'; + else throw std::domain_error("not an hex character"); }; for (size_t i=0; iself._infohash def __richcmp__(InfoHash self, InfoHash other, int op): From 0dccae68e62550009268c64d344aa0e3b1c12a86 Mon Sep 17 00:00:00 2001 From: Adrien Beraud Date: Fri, 3 Nov 2017 16:03:11 -0400 Subject: [PATCH 086/156] InfoHash: make false in case of parsing error --- include/opendht/infohash.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index 5acc96f8..e62193f0 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -288,10 +288,8 @@ std::istream& operator>> (std::istream& s, Hash& h) template Hash::Hash(const std::string& hex) { - if (hex.empty()) + if (hex.size() < 2*N) data_.fill(0); - else if (hex.size() < 2*N) - throw std::invalid_argument("string not long enough"); else fromString(hex.c_str()); } @@ -305,8 +303,11 @@ Hash::fromString(const char* in) { else if (c >= '0' and c <= '9') return c - '0'; else throw std::domain_error("not an hex character"); }; - for (size_t i=0; i Date: Fri, 3 Nov 2017 10:32:48 -0400 Subject: [PATCH 087/156] clean code --- CMakeLists.txt | 15 ++- README.md | 6 +- configure.ac | 12 +-- include/opendht.h | 3 + include/opendht/base64.h | 39 -------- .../{dhtproxyserver.h => dht_proxy_server.h} | 38 ++++++-- include/opendht/dhtrunner.h | 20 ---- include/opendht/value.h | 38 +------- src/Makefile.am | 4 +- src/base64.cpp | 43 +++++---- src/base64.h | 44 +++++++++ ...htproxyserver.cpp => dht_proxy_server.cpp} | 93 +++++++++++-------- src/dhtrunner.cpp | 16 ---- src/value.cpp | 40 +++++++- tools/Makefile.am | 6 -- tools/dhtnode.cpp | 20 ++-- 16 files changed, 224 insertions(+), 213 deletions(-) delete mode 100644 include/opendht/base64.h rename include/opendht/{dhtproxyserver.h => dht_proxy_server.h} (80%) create mode 100644 src/base64.h rename src/{dhtproxyserver.cpp => dht_proxy_server.cpp} (85%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 333b1a6b..4efc5af5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ option (OPENDHT_ARGON2 "Use included argon2 sources" OFF) option (OPENDHT_LTO "Build with LTO" OFF) option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF) option (OPENDHT_PROXY_SERVER "Enable DHT proxy server, use Restbed and jsoncpp" OFF) -option (OPENDHT_PROXY_SERVER_OPTIONAL "Enable sign and encrypt endpoints" OFF) +option (OPENDHT_PROXY_SERVER_IDENTITY "Allow clients to use the node identity" OFF) option (OPENDHT_PROXY_CLIENT "Enable DHT proxy client, use Restbed and jsoncpp" OFF) find_package(Doxygen) @@ -93,7 +93,6 @@ include_directories ( include/ include/opendht/ ${CMAKE_CURRENT_BINARY_DIR}/include/ - /usr/include/jsoncpp ) # Install dirs @@ -157,17 +156,17 @@ list (APPEND opendht_HEADERS if (OPENDHT_PROXY_SERVER) add_definitions(-DOPENDHT_PROXY_SERVER=true) - if (OPENDHT_PROXY_SERVER_OPTIONAL) - add_definitions(-DOPENDHT_PROXY_SERVER_OPTIONAL=true) + if (OPENDHT_PROXY_SERVER_IDENTITY) + add_definitions(-DOPENDHT_PROXY_SERVER_IDENTITY=true) else () - add_definitions(-DOPENDHT_PROXY_SERVER_OPTIONAL=false) + add_definitions(-DOPENDHT_PROXY_SERVER_IDENTITY=false) endif() list (APPEND opendht_HEADERS - include/opendht/dhtproxyserver.h - include/opendht/base64.h + include/opendht/dht_proxy_server.h ) list (APPEND opendht_SOURCES - src/dhtproxyserver.cpp + src/dht_proxy_server.cpp + src/base64.h src/base64.cpp ) else () diff --git a/README.md b/README.md index 885afaff..1ce85436 100644 --- a/README.md +++ b/README.md @@ -91,13 +91,11 @@ for r in results: - msgpack-c 1.2+, used for data serialization. - GnuTLS 3.3+, used for cryptographic operations. - Nettle 2.4+, a GnuTLS dependency for crypto. +- (optional) restbed 4.0+, used for the REST API. +- (optional) jsoncpp 1.7.4-3+, used for the REST API. - Build tested with GCC 4.8+ (GNU/Linux, Android, Windows with MinGW), Clang/LLVM (Linux, macOS). - Build tested with Microsoft Visual Studio 2015 -### Optional dependencies -- restbed 4.0+, used for the REST API. -- jsoncpp 1.7.4-3+, used for the REST API. - ## Contact IRC: join us on Freenode at [`#opendht`](https://webchat.freenode.net/?channels=%23opendht). diff --git a/configure.ac b/configure.ac index 6689c437..5863244b 100644 --- a/configure.ac +++ b/configure.ac @@ -135,13 +135,13 @@ AM_COND_IF([ENABLE_PROXY_SERVER], [ PKG_CHECK_MODULES([Jsoncpp], [jsoncpp >= 1.7.4]) CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=true -ljsoncpp -lrestbed" - AC_ARG_ENABLE([proxy_server_optionals], AS_HELP_STRING([--enable-proxy-server-optionals], - [Enable proxy server ability]), proxy_server_optionals=yes, proxy_server_optionals=no) - AM_CONDITIONAL(ENABLE_PROXY_SERVER_OPTIONALS, test x$proxy_server_optionals == xyes) - AM_COND_IF([ENABLE_PROXY_SERVER_OPTIONALS], [ - CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_OPTIONAL=true" + AC_ARG_ENABLE([proxy_server_identity], AS_HELP_STRING([--enable-proxy-server-identity], + [Enable proxy server ability]), proxy_server_identity=yes, proxy_server_identity=no) + AM_CONDITIONAL(ENABLE_PROXY_SERVER_IDENTITY, test x$proxy_server_identity == xyes) + AM_COND_IF([ENABLE_PROXY_SERVER_IDENTITY], [ + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=true" ], [ - CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_OPTIONAL=false" + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=false" ]) ], [ CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=false" diff --git a/include/opendht.h b/include/opendht.h index 98d3ece9..e003a216 100644 --- a/include/opendht.h +++ b/include/opendht.h @@ -19,6 +19,9 @@ #pragma once #include "opendht/dhtrunner.h" +#if OPENDHT_PROXY_SERVER +#include "opendht/dht_proxy_server.h" +#endif #include "opendht/log.h" #include "opendht/default_types.h" #include "opendht/indexation/pht.h" diff --git a/include/opendht/base64.h b/include/opendht/base64.h deleted file mode 100644 index 294762a8..00000000 --- a/include/opendht/base64.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2004-2017 Savoir-faire Linux Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#if OPENDHT_PROXY_SERVER -#pragma once - -#include -#include - -/** - * Encode a string into base64 - * @param begin - * @param end - * @return the encoded vector in a string - */ -std::string base64_encode(const std::vector& str); -/** - * Decode a base64 encoded string - * @param str to decode - * @return the decoded string - */ -std::string base64_decode(std::string const& str); - -#endif // OPENDHT_PROXY_SERVER diff --git a/include/opendht/dhtproxyserver.h b/include/opendht/dht_proxy_server.h similarity index 80% rename from include/opendht/dhtproxyserver.h rename to include/opendht/dht_proxy_server.h index 52d9f0e1..d614ecf0 100644 --- a/include/opendht/dhtproxyserver.h +++ b/include/opendht/dht_proxy_server.h @@ -21,6 +21,8 @@ #pragma once #include "def.h" +#include "sockaddr.h" +#include "infohash.h" #include #include @@ -29,6 +31,7 @@ namespace dht { class DhtRunner; +class InfoHash; /** * Describes the REST API @@ -43,13 +46,13 @@ public: * @note if the server fails to start (if port is already used or reserved), * it will fails silently */ - DhtProxyServer(DhtRunner* dht, unsigned int port = 8000); + DhtProxyServer(std::shared_ptr dht, in_port_t port = 8000); virtual ~DhtProxyServer(); - DhtProxyServer(const DhtProxyServer& other) = default; - DhtProxyServer(DhtProxyServer&& other) = default; - DhtProxyServer& operator=(const DhtProxyServer& other) = default; - DhtProxyServer& operator=(DhtProxyServer&& other) = default; + DhtProxyServer(const DhtProxyServer& other) = delete; + DhtProxyServer(DhtProxyServer&& other) = delete; + DhtProxyServer& operator=(const DhtProxyServer& other) = delete; + DhtProxyServer& operator=(DhtProxyServer&& other) = delete; /** * Stop the DhtProxyServer @@ -62,6 +65,7 @@ private: * Method: GET "/" * Result: HTTP 200, body: Value in JSON format (one part = one value) * On error: HTTP 404, body: {"err":"xxxx"} + * @param session */ void getNodeInfo(const std::shared_ptr& session) const; @@ -74,6 +78,7 @@ private: * {"ok": 1} * * On error: HTTP 404, body: {"err":"xxxx"} + * @param session */ void get(const std::shared_ptr& session) const; @@ -85,6 +90,7 @@ private: * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) * * On error: HTTP 404, body: {"err":"xxxx"} + * @param session */ void listen(const std::shared_ptr& session) const; @@ -95,10 +101,11 @@ private: * Return: {"ok":"1"} * On error: HTTP 404, body: {"err":"xxxx"} if no dht * HTTP 400, body: {"err":"xxxx"} if bad json + * @param session */ void put(const std::shared_ptr& session) const; -#if OPENDHT_PROXY_SERVER_OPTIONAL +#if OPENDHT_PROXY_SERVER_IDENTITY /** * Put a value to sign by the proxy on the DHT * Method: SIGN "/{InfoHash: .*}" @@ -106,6 +113,7 @@ private: * Return: {"ok":"1"} * On error: HTTP 404, body: {"err":"xxxx"} if no dht * HTTP 400, body: {"err":"xxxx"} if bad json + * @param session */ void putSigned(const std::shared_ptr& session) const; @@ -116,9 +124,10 @@ private: * Return: {"ok":"1"} * On error: HTTP 404, body: {"err":"xxxx"} if no dht * HTTP 400, body: {"err":"xxxx"} if bad json + * @param session */ void putEncrypted(const std::shared_ptr& session) const; -#endif // OPENDHT_PROXY_SERVER_OPTIONAL +#endif // OPENDHT_PROXY_SERVER_IDENTITY /** * Return Values of a InfoHash filtered by a value id @@ -129,13 +138,24 @@ private: * {"ok": 1} * * On error: HTTP 404, body: {"err":"xxxx"} + * @param session */ void getFiltered(const std::shared_ptr& session) const; std::thread server_thread {}; std::unique_ptr service_; - DhtRunner* dht_; - mutable std::vector> listenThreads_; + std::shared_ptr dht_; + + // Handle client quit for listen. + // NOTE: can be simplified when we will supports restbed 5.0 + std::thread listenThread_; + struct SessionToHashToken { + std::shared_ptr session; + InfoHash hash; + size_t token; + }; + mutable std::vector currentListeners_; + std::atomic_bool stopListeners {false}; }; } diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index acaac185..fd3346c0 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -36,10 +36,6 @@ #include #include -#if OPENDHT_PROXY_SERVER -#include "dhtproxyserver.h" -#endif - namespace dht { struct Node; @@ -366,18 +362,6 @@ public: */ void join(); -#if OPENDHT_PROXY_SERVER - /** - * Start the proxy server - * @param port used for the server - */ - void startProxyServer(unsigned int port=8000); - /** - * Stop the proxy server - */ - void stopProxyServer(); -#endif //OPENDHT_PROXY_SERVER - private: static constexpr std::chrono::seconds BOOTSTRAP_PERIOD {10}; @@ -407,10 +391,6 @@ private: std::mutex sock_mtx {}; std::vector> rcv {}; -#if OPENDHT_PROXY_SERVER - std::unique_ptr proxy_server_; -#endif //OPENDHT_PROXY_SERVER - /** true if currently actively boostraping */ std::atomic_bool bootstraping {false}; /* bootstrap nodes given as (host, service) pairs */ diff --git a/include/opendht/value.h b/include/opendht/value.h index e27e0f12..60dd1d99 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -39,7 +39,6 @@ #if OPENDHT_PROXY_SERVER #include -#include "base64.h" #endif //OPENDHT_PROXY_SERVER namespace dht { @@ -356,42 +355,7 @@ struct OPENDHT_PUBLIC Value * Build a value from a json object * @param json */ - Value(Json::Value& json) { - try { - if (json.isMember("id")) - id = ValueType::Id(json["id"].asInt()); - } catch (...) { } - if (json.isMember("cypher")) { - auto cypherStr = json["cypher"].asString(); - cypherStr = base64_decode(cypherStr); - cypher = std::vector(cypherStr.begin(), cypherStr.end()); - } - if (json.isMember("sig")) { - auto sigStr = json["sig"].asString(); - sigStr = base64_decode(sigStr); - signature = std::vector(sigStr.begin(), sigStr.end()); - } - if (json.isMember("seq")) - seq = json["seq"].asInt(); - if (json.isMember("owner")) { - auto ownerStr = json["owner"].asString(); - auto ownerBlob = std::vector(ownerStr.begin(), ownerStr.end()); - owner = std::make_shared(ownerBlob); - } - if (json.isMember("to")) { - auto toStr = json["to"].asString(); - recipient = InfoHash(toStr); - } - if (json.isMember("type")) - type = json["type"].asInt(); - if (json.isMember("data")){ - auto dataStr = json["data"].asString(); - dataStr = base64_decode(dataStr); - data = std::vector(dataStr.begin(), dataStr.end()); - } - if (json.isMember("utype")) - user_type = json["utype"].asString(); - } + Value(Json::Value& json); #endif //OPENDHT_PROXY_SERVER template diff --git a/src/Makefile.am b/src/Makefile.am index e9dd2e9f..4ff14741 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -55,8 +55,8 @@ nobase_include_HEADERS = \ ../include/opendht/indexation/pht.h if ENABLE_PROXY_SERVER -libopendht_la_SOURCES += base64.cpp dhtproxyserver.cpp -nobase_include_HEADERS += ../include/opendht/base64.h ../include/opendht/dhtproxyserver.h +libopendht_la_SOURCES += base64.h base64.cpp dht_proxy_server.cpp +nobase_include_HEADERS += ../include/opendht/dht_proxy_server.h endif clean-local: diff --git a/src/base64.cpp b/src/base64.cpp index 2732f40d..4971e9e0 100644 --- a/src/base64.cpp +++ b/src/base64.cpp @@ -1,21 +1,29 @@ /* - * Copyright (C) 2004-2017 Savoir-faire Linux Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ -#if OPENDHT_PROXY_SERVER + base64.cpp and base64.h + + Copyright (C) 2004-2008 René Nyffenegger + + This source code is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + + 3. This notice may not be removed or altered from any source distribution. + + René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +*/ #include "base64.h" static const std::string base64_chars = @@ -111,4 +119,3 @@ std::string base64_decode(std::string const& encoded_string) { return ret; } -#endif // OPENDHT_PROXY_SERVER diff --git a/src/base64.h b/src/base64.h new file mode 100644 index 00000000..ad3d2814 --- /dev/null +++ b/src/base64.h @@ -0,0 +1,44 @@ +/* + base64.cpp and base64.h + + Copyright (C) 2004-2008 René Nyffenegger + + This source code is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + + 3. This notice may not be removed or altered from any source distribution. + + René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +*/ +#pragma once + +#include +#include + +/** + * Encode a string into base64 + * @param begin + * @param end + * @return the encoded vector in a string + */ +std::string base64_encode(const std::vector& str); +/** + * Decode a base64 encoded string + * @param str to decode + * @return the decoded string + */ +std::string base64_decode(std::string const& str); diff --git a/src/dhtproxyserver.cpp b/src/dht_proxy_server.cpp similarity index 85% rename from src/dhtproxyserver.cpp rename to src/dht_proxy_server.cpp index 8557f874..ba6a998d 100644 --- a/src/dhtproxyserver.cpp +++ b/src/dht_proxy_server.cpp @@ -17,7 +17,7 @@ */ #if OPENDHT_PROXY_SERVER -#include "dhtproxyserver.h" +#include "dht_proxy_server.h" #include #include @@ -29,11 +29,10 @@ #include #include -#include - namespace dht { -DhtProxyServer::DhtProxyServer(DhtRunner* dht, unsigned int port) : dht_(dht) +DhtProxyServer::DhtProxyServer(std::shared_ptr dht, in_port_t port) +: dht_(dht) { // NOTE in c++14, use make_unique service_ = std::unique_ptr(new restbed::Service()); @@ -69,7 +68,7 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht, unsigned int port) : dht_(dht) this->put(session); } ); -#if OPENDHT_PROXY_SERVER_OPTIONAL +#if OPENDHT_PROXY_SERVER_IDENTITY resource->set_method_handler("SIGN", [this](const std::shared_ptr session) { @@ -82,7 +81,7 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht, unsigned int port) : dht_(dht) this->putEncrypted(session); } ); -#endif // OPENDHT_PROXY_SERVER_OPTIONAL +#endif // OPENDHT_PROXY_SERVER_IDENTITY service_->publish(resource); resource = std::make_shared(); resource->set_path("/{hash: .*}/{value: .*}"); @@ -106,6 +105,26 @@ DhtProxyServer::DhtProxyServer(DhtRunner* dht, unsigned int port) : dht_(dht) // Fail silently for now. } }); + + listenThread_ = std::thread([this]() { + auto stop = false; + while (!stop) { + auto listener = currentListeners_.begin(); + while (listener != currentListeners_.end()) { + if (listener->session->is_closed() && dht_) { + dht_->cancelListen(listener->hash, listener->token); + // Remove listener if unused + listener = currentListeners_.erase(listener); + } else { + ++listener; + } + } + //NOTE: When supports restbed 5.0: service_->is_up() and remove stopListeners + stop = stopListeners; + if (!stop) + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); } DhtProxyServer::~DhtProxyServer() @@ -117,10 +136,10 @@ void DhtProxyServer::stop() { service_->stop(); + stopListeners = true; // listenThreads_ will stop because there is no more sessions - for (auto& listenThread: listenThreads_) - if (listenThread->joinable()) - listenThread->join(); + if (listenThread_.joinable()) + listenThread_.join(); if (server_thread.joinable()) server_thread.join(); } @@ -131,7 +150,7 @@ DhtProxyServer::getNodeInfo(const std::shared_ptr& session) co const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& b) { (void)b; if (dht_) { @@ -152,17 +171,16 @@ DhtProxyServer::getNodeInfo(const std::shared_ptr& session) co void DhtProxyServer::get(const std::shared_ptr& session) const { - static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& b) { (void)b; if (dht_) { InfoHash infoHash(hash); - if (infoHash == INVALID_ID) { + if (!infoHash) { infoHash = InfoHash::get(hash); } Json::FastWriter writer; @@ -186,21 +204,20 @@ DhtProxyServer::get(const std::shared_ptr& session) const void DhtProxyServer::listen(const std::shared_ptr& session) const { - static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); InfoHash infoHash(hash); - if (infoHash == INVALID_ID) + if (!infoHash) infoHash = InfoHash::get(hash); - size_t token; session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& b) { (void)b; if (dht_) { + s->yield(restbed::OK, ""); // Open the connection Json::FastWriter writer; - token = dht_->listen(infoHash, [s, &writer](std::shared_ptr value) { + size_t token = dht_->listen(infoHash, [s, &writer](std::shared_ptr value) { // Send values as soon as we get them if (!s->is_closed()) s->yield(restbed::OK, writer.write(value->toJson())); @@ -208,14 +225,12 @@ DhtProxyServer::listen(const std::shared_ptr& session) const }).get(); // Handle client deconnection // NOTE: for now, there is no handler, so we test the session in a thread - listenThreads_.emplace_back(std::shared_ptr( - new std::thread([&]() { - while(!s->is_closed()) - std::this_thread::sleep_for(std::chrono::seconds(2)); - if (dht_) - dht_->cancelListen(infoHash, token); - })) - ); + // will be the case in restbed 5.0 + SessionToHashToken listener; + listener.session = s; + listener.hash = infoHash; + listener.token = token; + currentListeners_.emplace_back(listener); } else { s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); } @@ -228,16 +243,15 @@ void DhtProxyServer::put(const std::shared_ptr& session) const { // TODO test with the proxy client - static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); InfoHash infoHash(hash); - if (infoHash == INVALID_ID) + if (!infoHash) infoHash = InfoHash::get(hash); session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& b) { if (dht_) { if(b.empty()) { @@ -267,20 +281,19 @@ DhtProxyServer::put(const std::shared_ptr& session) const ); } -#if OPENDHT_PROXY_SERVER_OPTIONAL +#if OPENDHT_PROXY_SERVER_IDENTITY void DhtProxyServer::putSigned(const std::shared_ptr& session) const { - static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); InfoHash infoHash(hash); - if (infoHash == INVALID_ID) + if (!infoHash) infoHash = InfoHash::get(hash); session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& b) { if (dht_) { if(b.empty()) { @@ -312,16 +325,15 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons void DhtProxyServer::putEncrypted(const std::shared_ptr& session) const { - static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); InfoHash infoHash(hash); - if (infoHash == INVALID_ID) + if (!infoHash) infoHash = InfoHash::get(hash); session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& b) { if (dht_) { if(b.empty()) { @@ -338,7 +350,7 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c auto value = std::make_shared(root); auto toHash = request->get_path_parameter("to"); InfoHash toInfoHash(toHash); - if (toInfoHash == INVALID_ID) + if (toinfoHash) toInfoHash = InfoHash::get(toHash); auto response = value->toString(); @@ -357,23 +369,22 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c } ); } -#endif // OPENDHT_PROXY_SERVER_OPTIONAL +#endif // OPENDHT_PROXY_SERVER_IDENTITY void DhtProxyServer::getFiltered(const std::shared_ptr& session) const { - static constexpr dht::InfoHash INVALID_ID {}; const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); auto value = request->get_path_parameter("value"); session->fetch(content_length, - [&](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& b) { (void)b; if (dht_) { InfoHash infoHash(hash); - if (infoHash == INVALID_ID) { + if (!infoHash) { infoHash = InfoHash::get(hash); } Json::FastWriter writer; diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index 8988c3e8..66b4ebbd 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -800,20 +800,4 @@ DhtRunner::findCertificate(InfoHash hash, std::function(new DhtProxyServer(this, port)); -} - -void -DhtRunner::stopProxyServer() -{ - if (proxy_server_) - proxy_server_.reset(nullptr); -} -#endif //OPENDHT_PROXY_SERVER - } diff --git a/src/value.cpp b/src/value.cpp index a4e02bc4..759c17fb 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -23,7 +23,7 @@ #include "securedht.h" // print certificate ID #if OPENDHT_PROXY_SERVER -#include +#include "base64.h" #endif //OPENDHT_PROXY_SERVER @@ -173,6 +173,44 @@ Value::msgpack_unpack_body(const msgpack::object& o) } #if OPENDHT_PROXY_SERVER +Value::Value(Json::Value& json) +{ + try { + if (json.isMember("id")) + id = ValueType::Id(json["id"].asInt()); + } catch (...) { } + if (json.isMember("cypher")) { + auto cypherStr = json["cypher"].asString(); + cypherStr = base64_decode(cypherStr); + cypher = std::vector(cypherStr.begin(), cypherStr.end()); + } + if (json.isMember("sig")) { + auto sigStr = json["sig"].asString(); + sigStr = base64_decode(sigStr); + signature = std::vector(sigStr.begin(), sigStr.end()); + } + if (json.isMember("seq")) + seq = json["seq"].asInt(); + if (json.isMember("owner")) { + auto ownerStr = json["owner"].asString(); + auto ownerBlob = std::vector(ownerStr.begin(), ownerStr.end()); + owner = std::make_shared(ownerBlob); + } + if (json.isMember("to")) { + auto toStr = json["to"].asString(); + recipient = InfoHash(toStr); + } + if (json.isMember("type")) + type = json["type"].asInt(); + if (json.isMember("data")){ + auto dataStr = json["data"].asString(); + dataStr = base64_decode(dataStr); + data = std::vector(dataStr.begin(), dataStr.end()); + } + if (json.isMember("utype")) + user_type = json["utype"].asString(); +} + Json::Value Value::toJson() const { diff --git a/tools/Makefile.am b/tools/Makefile.am index 821ddce1..610fd76f 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -11,9 +11,3 @@ dhtchat_LDFLAGS = -lopendht -lreadline -L@top_builddir@/src/.libs @Argon2_LDFLAG dhtscanner_SOURCES = dhtscanner.cpp dhtscanner_LDFLAGS = -lopendht -lreadline -L@top_builddir@/src/.libs @Argon2_LDFLAGS@ @GnuTLS_LIBS@ - -if ENABLE_PROXY_SERVER -dhtnode_LDFLAGS += -lrestbed -ljsoncpp -dhtchat_LDFLAGS += -lrestbed -ljsoncpp -dhtscanner_LDFLAGS += -lrestbed -ljsoncpp -endif diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 31f845a0..ce0896b4 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -60,7 +60,7 @@ void print_help() { #if OPENDHT_PROXY_SERVER std::cout << std::endl << "Operations with the proxy:" << std::endl << " pst [port] Start the proxy interface on port." << std::endl - << " psp Stop the proxy interface." << std::endl; + << " psp [port] Stop the proxy interface on port." << std::endl; #endif //OPENDHT_PROXY_SERVER std::cout << std::endl << "Operations on the DHT:" << std::endl @@ -90,6 +90,9 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) #endif std::map indexes; +#if OPENDHT_PROXY_SERVER + std::map> proxies; +#endif //OPENDHT_PROXY_SERVER while (true) { @@ -174,13 +177,18 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) iss >> idstr; try { unsigned int port = std::stoi(idstr); - dht->startProxyServer(port); - } catch (...) { - dht->startProxyServer(); - } + auto proxyServer = std::unique_ptr( + new DhtProxyServer(dht, port) + ); + proxies.emplace(idstr, std::move(proxyServer)); + } catch (...) { } continue; } else if (op == "psp") { - dht->stopProxyServer(); + iss >> idstr; + auto it = proxies.find(idstr); + if (it != proxies.end()) { + it->second.reset(nullptr); + } continue; } #endif //OPENDHT_PROXY_SERVER From 67ba435ba685263052d985525a16ed04b3c2f69d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Mon, 6 Nov 2017 14:24:52 -0500 Subject: [PATCH 088/156] base64: take the old implementation from ring-daemon --- include/opendht/dht_proxy_server.h | 2 +- src/base64.cpp | 234 ++++++++++++++++------------- src/base64.h | 56 +++---- src/dht_proxy_server.cpp | 2 +- tools/dhtnode.cpp | 16 +- 5 files changed, 161 insertions(+), 149 deletions(-) diff --git a/include/opendht/dht_proxy_server.h b/include/opendht/dht_proxy_server.h index d614ecf0..e2b74525 100644 --- a/include/opendht/dht_proxy_server.h +++ b/include/opendht/dht_proxy_server.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Savoir-faire Linux Inc. + * Copyright (C) 2017 Savoir-faire Linux Inc. * Author : Sébastien Blin * * This program is free software; you can redistribute it and/or modify diff --git a/src/base64.cpp b/src/base64.cpp index 4971e9e0..2e95c80a 100644 --- a/src/base64.cpp +++ b/src/base64.cpp @@ -1,121 +1,143 @@ /* - base64.cpp and base64.h + * Copyright (C) 2004-2017 Savoir-faire Linux Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ - Copyright (C) 2004-2008 René Nyffenegger - - This source code is provided 'as-is', without any express or implied - warranty. In no event will the author be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. - - 3. This notice may not be removed or altered from any source distribution. - - René Nyffenegger rene.nyffenegger@adp-gmbh.ch - -*/ #include "base64.h" -static const std::string base64_chars = - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/"; +#include +#include +/* Mainly based on the following stackoverflow question: + * http://stackoverflow.com/questions/342409/how-do-i-base64-encode-decode-in-c + */ +static const char encoding_table[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', + '3', '4', '5', '6', '7', '8', '9', '+', '/' +}; -static inline bool is_base64(unsigned char c) { - return (isalnum(c) || (c == '+') || (c == '/')); +static const size_t mod_table[] = { 0, 2, 1 }; + +char *base64_encode(const uint8_t *input, size_t input_length, + char *output, size_t *output_length) +{ + size_t i, j; + size_t out_sz = *output_length; + *output_length = 4 * ((input_length + 2) / 3); + if (out_sz < *output_length || output == nullptr) + return nullptr; + + for (i = 0, j = 0; i < input_length; ) { + uint8_t octet_a = i < input_length ? input[i++] : 0; + uint8_t octet_b = i < input_length ? input[i++] : 0; + uint8_t octet_c = i < input_length ? input[i++] : 0; + + uint32_t triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; + + output[j++] = encoding_table[(triple >> 3 * 6) & 0x3F]; + output[j++] = encoding_table[(triple >> 2 * 6) & 0x3F]; + output[j++] = encoding_table[(triple >> 1 * 6) & 0x3F]; + output[j++] = encoding_table[(triple >> 0 * 6) & 0x3F]; + } + + for (i = 0; i < mod_table[input_length % 3]; i++) + output[*output_length - 1 - i] = '='; + + return output; } -std::string base64_encode(const std::vector& str) { - unsigned char const* bytes_to_encode = str.data(); - unsigned int in_len = str.size(); - std::string ret; - int i = 0; - int j = 0; - unsigned char char_array_3[3]; - unsigned char char_array_4[4]; +uint8_t *base64_decode(const char *input, size_t input_length, + uint8_t *output, size_t *output_length) +{ + size_t i, j; + uint8_t decoding_table[256]; - while (in_len--) { - char_array_3[i++] = *(bytes_to_encode++); - if (i == 3) { - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; + uint8_t c; + for (c = 0; c < 64; c++) + decoding_table[static_cast(encoding_table[c])] = c; - for(i = 0; (i <4) ; i++) - ret += base64_chars[char_array_4[i]]; - i = 0; - } + if (input_length % 4 != 0) + return nullptr; + + size_t out_sz = *output_length; + *output_length = input_length / 4 * 3; + if (input[input_length - 1] == '=') + (*output_length)--; + if (input[input_length - 2] == '=') + (*output_length)--; + + if (out_sz < *output_length || output == nullptr) + return nullptr; + + for (i = 0, j = 0; i < input_length;) { + uint8_t sextet_a = input[i] == '=' ? 0 & i++ + : decoding_table[static_cast(input[i++])]; + uint8_t sextet_b = input[i] == '=' ? 0 & i++ + : decoding_table[static_cast(input[i++])]; + uint8_t sextet_c = input[i] == '=' ? 0 & i++ + : decoding_table[static_cast(input[i++])]; + uint8_t sextet_d = input[i] == '=' ? 0 & i++ + : decoding_table[static_cast(input[i++])]; + + uint32_t triple = (sextet_a << 3 * 6) + + (sextet_b << 2 * 6) + + (sextet_c << 1 * 6) + + (sextet_d << 0 * 6); + + if (j < *output_length) + output[j++] = (triple >> 2 * 8) & 0xFF; + if (j < *output_length) + output[j++] = (triple >> 1 * 8) & 0xFF; + if (j < *output_length) + output[j++] = (triple >> 0 * 8) & 0xFF; } - if (i) - { - for(j = i; j < 3; j++) - char_array_3[j] = '\0'; - - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; - - for (j = 0; (j < i + 1); j++) - ret += base64_chars[char_array_4[j]]; - - while((i++ < 3)) - ret += '='; - } - - return ret; + return output; } -std::string base64_decode(std::string const& encoded_string) { - size_t in_len = encoded_string.size(); - size_t i = 0; - size_t j = 0; - int in_ = 0; - unsigned char char_array_4[4], char_array_3[3]; - std::string ret; - - while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { - char_array_4[i++] = encoded_string[in_]; in_++; - if (i ==4) { - for (i = 0; i <4; i++) - char_array_4[i] = static_cast(base64_chars.find(char_array_4[i])); - - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - for (i = 0; (i < 3); i++) - ret += char_array_3[i]; - i = 0; - } - } - - if (i) { - for (j = i; j <4; j++) - char_array_4[j] = 0; - - for (j = 0; j <4; j++) - char_array_4[j] = static_cast(base64_chars.find(char_array_4[j])); - - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; - } - - return ret; +std::string +base64_encode(const std::vector::const_iterator begin, + const std::vector::const_iterator end) +{ + size_t output_length = 4 * ((std::distance(begin, end) + 2) / 3); + std::string out; + out.resize(output_length); + base64_encode(&(*begin), std::distance(begin, end), + &(*out.begin()), &output_length); + out.resize(output_length); + return out; +} + + +std::string +base64_encode(const std::vector& str) +{ + return base64_encode(str.cbegin(), str.cend()); +} + +std::string +base64_decode(const std::string& str) +{ + size_t output_length = str.length() / 4 * 3 + 2; + std::vector output; + output.resize(output_length); + base64_decode(str.data(), str.size(), output.data(), &output_length); + output.resize(output_length); + return std::string(output.begin(), output.end()); } diff --git a/src/base64.h b/src/base64.h index ad3d2814..3f5f39e1 100644 --- a/src/base64.h +++ b/src/base64.h @@ -1,44 +1,36 @@ /* - base64.cpp and base64.h + * Copyright (C) 2004-2017 Savoir-faire Linux Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ - Copyright (C) 2004-2008 René Nyffenegger - - This source code is provided 'as-is', without any express or implied - warranty. In no event will the author be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. - - 3. This notice may not be removed or altered from any source distribution. - - René Nyffenegger rene.nyffenegger@adp-gmbh.ch - -*/ #pragma once #include #include /** - * Encode a string into base64 - * @param begin - * @param end - * @return the encoded vector in a string + * Encode a buffer in base64. + * + * @param str the input buffer + * @return a base64-encoded buffer */ std::string base64_encode(const std::vector& str); /** - * Decode a base64 encoded string - * @param str to decode - * @return the decoded string + * Decode a buffer in base64. + * + * @param str the input buffer + * @return a base64-decoded buffer */ -std::string base64_decode(std::string const& str); +std::string base64_decode(const std::string& str); diff --git a/src/dht_proxy_server.cpp b/src/dht_proxy_server.cpp index ba6a998d..9f70e2bf 100644 --- a/src/dht_proxy_server.cpp +++ b/src/dht_proxy_server.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Savoir-faire Linux Inc. + * Copyright (C) 2017 Savoir-faire Linux Inc. * Author : Sébastien Blin * * This program is free software; you can redistribute it and/or modify diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index ce0896b4..0c18af27 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -91,7 +91,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) std::map indexes; #if OPENDHT_PROXY_SERVER - std::map> proxies; + std::map> proxies; #endif //OPENDHT_PROXY_SERVER while (true) @@ -177,18 +177,16 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) iss >> idstr; try { unsigned int port = std::stoi(idstr); - auto proxyServer = std::unique_ptr( - new DhtProxyServer(dht, port) - ); - proxies.emplace(idstr, std::move(proxyServer)); + proxies.emplace(port, new DhtProxyServer(dht, port)); } catch (...) { } continue; } else if (op == "psp") { iss >> idstr; - auto it = proxies.find(idstr); - if (it != proxies.end()) { - it->second.reset(nullptr); - } + try { + auto it = proxies.find(std::stoi(idstr)); + if (it != proxies.end()) + proxies.erase(it); + } catch (...) { } continue; } #endif //OPENDHT_PROXY_SERVER From 38ad5b9ef2a9ae707b0546d18da44c4f56ec0015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Mon, 6 Nov 2017 16:49:46 -0500 Subject: [PATCH 089/156] add --proxyserver option for dhtnode --- tools/dhtnode.cpp | 5 ++++- tools/tools_common.h | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 0c18af27..d8c81a9a 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -29,7 +29,7 @@ extern "C" { using namespace dht; void print_usage() { - std::cout << "Usage: dhtnode [-v [-l logfile]] [-i] [-d] [-n network_id] [-p local_port] [-b bootstrap_host[:port]]" << std::endl << std::endl; + std::cout << "Usage: dhtnode [-v [-l logfile]] [-i] [-d] [-n network_id] [-p local_port] [-b bootstrap_host[:port]] [--proxyserver local_port]" << std::endl << std::endl; std::cout << "dhtnode, a simple OpenDHT command line node runner." << std::endl; std::cout << "Report bugs to: http://opendht.net" << std::endl; } @@ -92,6 +92,9 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) std::map indexes; #if OPENDHT_PROXY_SERVER std::map> proxies; + if (params.proxyserver != 0) { + proxies.emplace(params.proxyserver, new DhtProxyServer(dht, params.proxyserver)); + } #endif //OPENDHT_PROXY_SERVER while (true) diff --git a/tools/tools_common.h b/tools/tools_common.h index 933cc7e2..cd3e1970 100644 --- a/tools/tools_common.h +++ b/tools/tools_common.h @@ -121,6 +121,7 @@ struct dht_params { bool daemonize {false}; bool service {false}; std::pair bootstrap {}; + in_port_t proxyserver {0}; }; static const constexpr struct option long_options[] = { @@ -134,6 +135,7 @@ static const constexpr struct option long_options[] = { {"service", no_argument , nullptr, 's'}, {"logfile", required_argument, nullptr, 'l'}, {"syslog", no_argument , nullptr, 'L'}, + {"proxyserver",required_argument, nullptr, 'S'}, {nullptr, 0 , nullptr, 0} }; @@ -151,6 +153,14 @@ parseArgs(int argc, char **argv) { std::cout << "Invalid port: " << port_arg << std::endl; } break; + case 'S': { + int port_arg = atoi(optarg); + if (port_arg >= 0 && port_arg < 0x10000) + params.proxyserver = port_arg; + else + std::cout << "Invalid port: " << port_arg << std::endl; + } + break; case 'n': params.network = strtoul(optarg, nullptr, 0); break; From bbe922dd7d6092e86f555e174cd1cf474c1fd49b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Tue, 7 Nov 2017 13:30:06 -0500 Subject: [PATCH 090/156] remove headers, fix HTTP status, fix get / --- include/opendht/callbacks.h | 10 +++ include/opendht/dht_proxy_server.h | 41 ++++----- src/callbacks.cpp | 20 +++++ src/dht_proxy_server.cpp | 134 +++++++++++++++-------------- 4 files changed, 120 insertions(+), 85 deletions(-) diff --git a/include/opendht/callbacks.h b/include/opendht/callbacks.h index 21f2dd0c..e354ba21 100644 --- a/include/opendht/callbacks.h +++ b/include/opendht/callbacks.h @@ -26,6 +26,10 @@ #include #include +#if OPENDHT_PROXY_SERVER +#include +#endif //OPENDHT_PROXY_SERVER + namespace dht { struct Node; @@ -47,6 +51,12 @@ struct OPENDHT_PUBLIC NodeStats { unsigned table_depth; unsigned getKnownNodes() const { return good_nodes + dubious_nodes; } std::string toString() const; +#if OPENDHT_PROXY_SERVER + /** + * Build a json object from a NodeStats + */ + Json::Value toJson() const; +#endif //OPENDHT_PROXY_SERVER }; /** diff --git a/include/opendht/dht_proxy_server.h b/include/opendht/dht_proxy_server.h index e2b74525..5ac64c4e 100644 --- a/include/opendht/dht_proxy_server.h +++ b/include/opendht/dht_proxy_server.h @@ -31,7 +31,6 @@ namespace dht { class DhtRunner; -class InfoHash; /** * Describes the REST API @@ -64,7 +63,7 @@ private: * Return the PublicKey id, the node id and node stats * Method: GET "/" * Result: HTTP 200, body: Value in JSON format (one part = one value) - * On error: HTTP 404, body: {"err":"xxxx"} + * On error: HTTP 503, body: {"err":"xxxx"} * @param session */ void getNodeInfo(const std::shared_ptr& session) const; @@ -72,12 +71,11 @@ private: /** * Return Values of a InfoHash * Method: GET "/{InfoHash: .*}" - * Return: Multiple JSON object in parts. For 2 values, you will have 3 parts: - * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) - * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) - * {"ok": 1} + * Return: Multiple JSON object in parts. Example: + * Value in JSON format\n + * Value in JSON format * - * On error: HTTP 404, body: {"err":"xxxx"} + * On error: HTTP 503, body: {"err":"xxxx"} * @param session */ void get(const std::shared_ptr& session) const; @@ -85,11 +83,11 @@ private: /** * Listen incoming Values of a InfoHash. * Method: LISTEN "/{InfoHash: .*}" - * Return: Multiple JSON object in parts. For 2 values, you will have 2 parts: - * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) - * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) + * Return: Multiple JSON object in parts. Example: + * Value in JSON format\n + * Value in JSON format * - * On error: HTTP 404, body: {"err":"xxxx"} + * On error: HTTP 503, body: {"err":"xxxx"} * @param session */ void listen(const std::shared_ptr& session) const; @@ -98,8 +96,8 @@ private: * Put a value on the DHT * Method: POST "/{InfoHash: .*}" * body = Value to put in JSON - * Return: {"ok":"1"} - * On error: HTTP 404, body: {"err":"xxxx"} if no dht + * Return: HTTP 200 if success and the value put in JSON + * On error: HTTP 503, body: {"err":"xxxx"} if no dht * HTTP 400, body: {"err":"xxxx"} if bad json * @param session */ @@ -110,8 +108,8 @@ private: * Put a value to sign by the proxy on the DHT * Method: SIGN "/{InfoHash: .*}" * body = Value to put in JSON - * Return: {"ok":"1"} - * On error: HTTP 404, body: {"err":"xxxx"} if no dht + * Return: HTTP 200 if success and the value put in JSON + * On error: HTTP 503, body: {"err":"xxxx"} if no dht * HTTP 400, body: {"err":"xxxx"} if bad json * @param session */ @@ -121,8 +119,8 @@ private: * Put a value to encrypt by the proxy on the DHT * Method: ENCRYPT "/{hash: .*}" * body = Value to put in JSON + "to":"infoHash" - * Return: {"ok":"1"} - * On error: HTTP 404, body: {"err":"xxxx"} if no dht + * Return: HTTP 200 if success and the value put in JSON + * On error: HTTP 503, body: {"err":"xxxx"} if no dht * HTTP 400, body: {"err":"xxxx"} if bad json * @param session */ @@ -132,12 +130,11 @@ private: /** * Return Values of a InfoHash filtered by a value id * Method: GET "/{InfoHash: .*}/{ValueId: .*}" - * Return: Multiple JSON object in parts. For 2 values, you will have 3 parts: - * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) - * Value in JSON format (HTTP/1.1 200 OK Content-Type: application/json) - * {"ok": 1} + * Return: Multiple JSON object in parts. Example: + * Value in JSON format\n + * Value in JSON format * - * On error: HTTP 404, body: {"err":"xxxx"} + * On error: HTTP 503, body: {"err":"xxxx"} * @param session */ void getFiltered(const std::shared_ptr& session) const; diff --git a/src/callbacks.cpp b/src/callbacks.cpp index 8908a2df..89719ef1 100644 --- a/src/callbacks.cpp +++ b/src/callbacks.cpp @@ -68,4 +68,24 @@ NodeStats::toString() const return ss.str(); } +#if OPENDHT_PROXY_SERVER +/** + * Build a json object from a NodeStats + */ +Json::Value +NodeStats::toJson() const +{ + Json::Value val; + val["good"] = static_cast(good_nodes); + val["dubious"] = static_cast(dubious_nodes); + val["incoming"] = static_cast(incoming_nodes); + if (table_depth > 1) { + val["table_depth"] = static_cast(table_depth); + unsigned long tot_nodes = 8 * std::exp2(table_depth); + val["network_size_estimation"] = static_cast(tot_nodes); + } + return val; +} +#endif //OPENDHT_PROXY_SERVER + } diff --git a/src/dht_proxy_server.cpp b/src/dht_proxy_server.cpp index 9f70e2bf..f813f239 100644 --- a/src/dht_proxy_server.cpp +++ b/src/dht_proxy_server.cpp @@ -19,13 +19,12 @@ #if OPENDHT_PROXY_SERVER #include "dht_proxy_server.h" -#include -#include - #include "default_types.h" #include "dhtrunner.h" #include "msgpack.hpp" +#include +#include #include #include @@ -96,13 +95,14 @@ DhtProxyServer::DhtProxyServer(std::shared_ptr dht, in_port_t port) // Start server auto settings = std::make_shared(); settings->set_default_header("Content-Type", "application/json"); + settings->set_default_header("Connection", "keep-alive"); std::chrono::milliseconds timeout(std::numeric_limits::max()); settings->set_connection_timeout(timeout); // there is a timeout, but really huge settings->set_port(port); try { service_->start(settings); } catch(std::system_error& e) { - // Fail silently for now. + std::cerr << "Error running server on port " << port << ": " << e.what() << std::endl; } }); @@ -150,20 +150,21 @@ DhtProxyServer::getNodeInfo(const std::shared_ptr& session) co const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); session->fetch(content_length, - [=](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& /*b*/) { - (void)b; if (dht_) { Json::Value result; - result["id"] = dht_->getId().toString(); + auto id = dht_->getId(); + if (id) + result["id"] = id.toString(); result["node_id"] = dht_->getNodeId().toString(); - result["ipv4"] = dht_->getNodesStats(AF_INET).toString(); - result["ipv6"] = dht_->getNodesStats(AF_INET6).toString(); + result["ipv4"] = dht_->getNodesStats(AF_INET).toJson(); + result["ipv6"] = dht_->getNodesStats(AF_INET6).toJson(); Json::FastWriter writer; s->close(restbed::OK, writer.write(result)); } else - s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); } ); } @@ -175,35 +176,36 @@ DhtProxyServer::get(const std::shared_ptr& session) const int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); session->fetch(content_length, - [=](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& /*b* */) { - (void)b; if (dht_) { InfoHash infoHash(hash); if (!infoHash) { infoHash = InfoHash::get(hash); } - Json::FastWriter writer; - dht_->get(infoHash, [s, &writer](std::shared_ptr value) { - // Send values as soon as we get them - Json::Value result; - s->yield(restbed::OK, writer.write(value->toJson())); - return true; - }, [s, &writer](bool ok) { - // Communication is finished - auto response = std::to_string(ok); - s->close(restbed::OK, "{\"ok\": " + response + "}"); + s->yield(restbed::OK, "", [=]( const std::shared_ptr< restbed::Session > s) { + dht_->get(infoHash, [s](std::shared_ptr value) { + // Send values as soon as we get them + Json::FastWriter writer; + s->yield(writer.write(value->toJson()), [](const std::shared_ptr /*session*/){ }); + return true; + }, [s](bool /*ok* */) { + // Communication is finished + s->close(); + }); }); } else { - s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); + } void DhtProxyServer::listen(const std::shared_ptr& session) const { + const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); @@ -211,28 +213,33 @@ DhtProxyServer::listen(const std::shared_ptr& session) const if (!infoHash) infoHash = InfoHash::get(hash); session->fetch(content_length, - [=](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& /*b* */) { - (void)b; if (dht_) { - s->yield(restbed::OK, ""); // Open the connection - Json::FastWriter writer; - size_t token = dht_->listen(infoHash, [s, &writer](std::shared_ptr value) { - // Send values as soon as we get them - if (!s->is_closed()) - s->yield(restbed::OK, writer.write(value->toJson())); - return !s->is_closed(); - }).get(); - // Handle client deconnection - // NOTE: for now, there is no handler, so we test the session in a thread - // will be the case in restbed 5.0 - SessionToHashToken listener; - listener.session = s; - listener.hash = infoHash; - listener.token = token; - currentListeners_.emplace_back(listener); + InfoHash infoHash(hash); + if (!infoHash) { + infoHash = InfoHash::get(hash); + } + s->yield(restbed::OK, "", [=]( const std::shared_ptr< restbed::Session > s) { + size_t token = dht_->listen(infoHash, [s](std::shared_ptr value) { + // Send values as soon as we get them + if (!s->is_closed()) { + Json::FastWriter writer; + s->yield(writer.write(value->toJson()), [](const std::shared_ptr /*session*/){ }); + } + return !s->is_closed(); + }).get(); + // Handle client deconnection + // NOTE: for now, there is no handler, so we test the session in a thread + // will be the case in restbed 5.0 + SessionToHashToken listener; + listener.session = s; + listener.hash = infoHash; + listener.token = token; + currentListeners_.emplace_back(listener); + }); } else { - s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); @@ -267,15 +274,15 @@ DhtProxyServer::put(const std::shared_ptr& session) const // Build the Value from json auto value = std::make_shared(root); - auto response = value->toString(); + Json::FastWriter writer; dht_->put(infoHash, value); - s->close(restbed::OK, response); + s->close(restbed::OK, writer.write(value->toJson())); } else { s->close(restbed::BAD_REQUEST, "Incorrect JSON"); } } } else { - s->close(restbed::NOT_FOUND, ""); + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); @@ -308,15 +315,15 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons if (parsingSuccessful) { auto value = std::make_shared(root); - auto response = value->toString(); + Json::FastWriter writer; dht_->putSigned(infoHash, value); - s->close(restbed::OK, response); + s->close(restbed::OK, writer.write(value->toJson())); } else { s->close(restbed::BAD_REQUEST, "Incorrect JSON" + strJson); } } } else { - s->close(restbed::NOT_FOUND, ""); + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); @@ -353,9 +360,9 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c if (toinfoHash) toInfoHash = InfoHash::get(toHash); - auto response = value->toString(); + Json::FastWriter writer; dht_->putEncrypted(infoHash, toInfoHash, value); - s->close(restbed::OK, response); + s->close(restbed::OK, writer.write(value->toJson())); } else { if(!parsingSuccessful) s->close(restbed::BAD_REQUEST, "Incorrect JSON"); @@ -364,7 +371,7 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c } } } else { - s->close(restbed::NOT_FOUND, ""); + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); @@ -379,25 +386,26 @@ DhtProxyServer::getFiltered(const std::shared_ptr& session) co auto hash = request->get_path_parameter("hash"); auto value = request->get_path_parameter("value"); session->fetch(content_length, - [=](const std::shared_ptr s, const restbed::Bytes& b) + [=](const std::shared_ptr s, const restbed::Bytes& /*b* */) { - (void)b; if (dht_) { InfoHash infoHash(hash); if (!infoHash) { infoHash = InfoHash::get(hash); } - Json::FastWriter writer; - dht_->get(infoHash, [s, &writer](std::shared_ptr value) { - Json::Value result; - s->yield(restbed::OK, writer.write(value->toJson())); - return true; - }, [s, &writer](bool ok) { - auto response = std::to_string(ok); - s->close(restbed::OK, "{\"ok\": " + response + "}"); - }, {}, value); + s->yield(restbed::OK, "", [=]( const std::shared_ptr< restbed::Session > s) { + dht_->get(infoHash, [s](std::shared_ptr v) { + // Send values as soon as we get them + Json::FastWriter writer; + s->yield(writer.write(v->toJson()), [](const std::shared_ptr /*session*/){ }); + return true; + }, [s](bool /*ok* */) { + // Communication is finished + s->close(); + }, {}, value); + }); } else { - s->close(restbed::NOT_FOUND, "{\"err\":\"Incorrect DhtRunner\"}"); + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); From 2ca2835c9b73e60f9e184a8dd50faea5633a9574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 8 Nov 2017 09:43:42 -0500 Subject: [PATCH 091/156] add options method handler --- include/opendht/dht_proxy_server.h | 8 ++++++++ src/dht_proxy_server.cpp | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/include/opendht/dht_proxy_server.h b/include/opendht/dht_proxy_server.h index 5ac64c4e..733ebdb2 100644 --- a/include/opendht/dht_proxy_server.h +++ b/include/opendht/dht_proxy_server.h @@ -139,6 +139,14 @@ private: */ void getFiltered(const std::shared_ptr& session) const; + /** + * Respond allowed Methods + * Method: OPTIONS "/{hash: .*}" + * Return: HTTP 200 + Allow: allowed methods + * See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS + */ + void handleOptionsMethod(const std::shared_ptr& session) const; + std::thread server_thread {}; std::unique_ptr service_; std::shared_ptr dht_; diff --git a/src/dht_proxy_server.cpp b/src/dht_proxy_server.cpp index f813f239..2b2e34ec 100644 --- a/src/dht_proxy_server.cpp +++ b/src/dht_proxy_server.cpp @@ -81,6 +81,12 @@ DhtProxyServer::DhtProxyServer(std::shared_ptr dht, in_port_t port) } ); #endif // OPENDHT_PROXY_SERVER_IDENTITY + resource->set_method_handler("OPTIONS", + [this](const std::shared_ptr session) + { + this->handleOptionsMethod(session); + } + ); service_->publish(resource); resource = std::make_shared(); resource->set_path("/{hash: .*}/{value: .*}"); @@ -357,7 +363,7 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c auto value = std::make_shared(root); auto toHash = request->get_path_parameter("to"); InfoHash toInfoHash(toHash); - if (toinfoHash) + if (toInfoHash) toInfoHash = InfoHash::get(toHash); Json::FastWriter writer; @@ -378,6 +384,19 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c } #endif // OPENDHT_PROXY_SERVER_IDENTITY +void +DhtProxyServer::handleOptionsMethod(const std::shared_ptr& session) const +{ +#if OPENDHT_PROXY_SERVER_IDENTITY + const auto allowed = "OPTIONS, GET, POST, LISTEN, SIGN, ENCRYPT"; +#else + const auto allowed = "OPTIONS, GET, POST, LISTEN"; +#endif //OPENDHT_PROXY_SERVER_IDENTITY + session->close(restbed::OK, {{"Access-Control-Allow-Methods", allowed}, + {"Access-Control-Allow-Headers", "content-type"}, + {"Access-Control-Max-Age", "86400"}}); +} + void DhtProxyServer::getFiltered(const std::shared_ptr& session) const { From e2c534df5ababd4504918bf1aa0b6caf71ff62c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 8 Nov 2017 14:06:24 -0500 Subject: [PATCH 092/156] add std::bind, fix put, avoid overflow on base64 --- include/opendht/dht_proxy_server.h | 6 +- src/base64.cpp | 2 +- src/dht_proxy_server.cpp | 132 ++++++++++------------------- 3 files changed, 49 insertions(+), 91 deletions(-) diff --git a/include/opendht/dht_proxy_server.h b/include/opendht/dht_proxy_server.h index 733ebdb2..d2eb6a38 100644 --- a/include/opendht/dht_proxy_server.h +++ b/include/opendht/dht_proxy_server.h @@ -62,7 +62,7 @@ private: /** * Return the PublicKey id, the node id and node stats * Method: GET "/" - * Result: HTTP 200, body: Value in JSON format (one part = one value) + * Result: HTTP 200, body: Node infos in JSON format * On error: HTTP 503, body: {"err":"xxxx"} * @param session */ @@ -98,7 +98,7 @@ private: * body = Value to put in JSON * Return: HTTP 200 if success and the value put in JSON * On error: HTTP 503, body: {"err":"xxxx"} if no dht - * HTTP 400, body: {"err":"xxxx"} if bad json + * HTTP 400, body: {"err":"xxxx"} if bad json or HTTP 502 if put fails * @param session */ void put(const std::shared_ptr& session) const; @@ -157,7 +157,7 @@ private: struct SessionToHashToken { std::shared_ptr session; InfoHash hash; - size_t token; + std::future token; }; mutable std::vector currentListeners_; std::atomic_bool stopListeners {false}; diff --git a/src/base64.cpp b/src/base64.cpp index 2e95c80a..7e255582 100644 --- a/src/base64.cpp +++ b/src/base64.cpp @@ -72,7 +72,7 @@ uint8_t *base64_decode(const char *input, size_t input_length, for (c = 0; c < 64; c++) decoding_table[static_cast(encoding_table[c])] = c; - if (input_length % 4 != 0) + if (input_length % 4 != 0 || input_length < 2) return nullptr; size_t out_sz = *output_length; diff --git a/src/dht_proxy_server.cpp b/src/dht_proxy_server.cpp index 2b2e34ec..586a8b77 100644 --- a/src/dht_proxy_server.cpp +++ b/src/dht_proxy_server.cpp @@ -28,6 +28,8 @@ #include #include +using namespace std::placeholders; + namespace dht { DhtProxyServer::DhtProxyServer(std::shared_ptr dht, in_port_t port) @@ -40,62 +42,22 @@ DhtProxyServer::DhtProxyServer(std::shared_ptr dht, in_port_t port) // Create endpoints auto resource = std::make_shared(); resource->set_path("/"); - resource->set_method_handler("GET", - [this](const std::shared_ptr session) - { - this->getNodeInfo(session); - } - ); + resource->set_method_handler("GET", std::bind(&DhtProxyServer::getNodeInfo, this, _1)); service_->publish(resource); resource = std::make_shared(); resource->set_path("/{hash: .*}"); - resource->set_method_handler("GET", - [this](const std::shared_ptr session) - { - this->get(session); - } - ); - resource->set_method_handler("LISTEN", - [this](const std::shared_ptr session) - { - this->listen(session); - } - ); - resource->set_method_handler("POST", - [this](const std::shared_ptr session) - { - this->put(session); - } - ); + resource->set_method_handler("GET", std::bind(&DhtProxyServer::get, this, _1)); + resource->set_method_handler("LISTEN", std::bind(&DhtProxyServer::listen, this, _1)); + resource->set_method_handler("POST", std::bind(&DhtProxyServer::put, this, _1)); #if OPENDHT_PROXY_SERVER_IDENTITY - resource->set_method_handler("SIGN", - [this](const std::shared_ptr session) - { - this->putSigned(session); - } - ); - resource->set_method_handler("ENCRYPT", - [this](const std::shared_ptr session) - { - this->putEncrypted(session); - } - ); + resource->set_method_handler("SIGN", std::bind(&DhtProxyServer::putSigned, this, _1)); + resource->set_method_handler("ENCRYPT", std::bind(&DhtProxyServer::putEncrypted, this, _1)); #endif // OPENDHT_PROXY_SERVER_IDENTITY - resource->set_method_handler("OPTIONS", - [this](const std::shared_ptr session) - { - this->handleOptionsMethod(session); - } - ); + resource->set_method_handler("OPTIONS", std::bind(&DhtProxyServer::handleOptionsMethod, this, _1)); service_->publish(resource); resource = std::make_shared(); resource->set_path("/{hash: .*}/{value: .*}"); - resource->set_method_handler("GET", - [this](const std::shared_ptr session) - { - this->getFiltered(session); - } - ); + resource->set_method_handler("GET", std::bind(&DhtProxyServer::getFiltered, this, _1)); service_->publish(resource); // Start server @@ -118,7 +80,7 @@ DhtProxyServer::DhtProxyServer(std::shared_ptr dht, in_port_t port) auto listener = currentListeners_.begin(); while (listener != currentListeners_.end()) { if (listener->session->is_closed() && dht_) { - dht_->cancelListen(listener->hash, listener->token); + dht_->cancelListen(listener->hash, std::move(listener->token)); // Remove listener if unused listener = currentListeners_.erase(listener); } else { @@ -226,36 +188,32 @@ DhtProxyServer::listen(const std::shared_ptr& session) const if (!infoHash) { infoHash = InfoHash::get(hash); } - s->yield(restbed::OK, "", [=]( const std::shared_ptr< restbed::Session > s) { - size_t token = dht_->listen(infoHash, [s](std::shared_ptr value) { - // Send values as soon as we get them - if (!s->is_closed()) { - Json::FastWriter writer; - s->yield(writer.write(value->toJson()), [](const std::shared_ptr /*session*/){ }); - } - return !s->is_closed(); - }).get(); + s->yield(restbed::OK); // Handle client deconnection // NOTE: for now, there is no handler, so we test the session in a thread // will be the case in restbed 5.0 - SessionToHashToken listener; - listener.session = s; - listener.hash = infoHash; - listener.token = token; - currentListeners_.emplace_back(listener); + SessionToHashToken listener; + listener.session = session; + listener.hash = infoHash; + listener.token = dht_->listen(infoHash, [s](std::shared_ptr value) { + // Send values as soon as we get them + if (!s->is_closed()) { + Json::FastWriter writer; + s->yield(writer.write(value->toJson()), [](const std::shared_ptr /*session*/){ }); + } + return !s->is_closed(); }); + currentListeners_.emplace_back(std::move(listener)); } else { - s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); + session->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); } } ); } - void DhtProxyServer::put(const std::shared_ptr& session) const { - // TODO test with the proxy client const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); @@ -268,7 +226,7 @@ DhtProxyServer::put(const std::shared_ptr& session) const { if (dht_) { if(b.empty()) { - std::string response("Missing parameters"); + std::string response("{\"err\":\"Missing parameters\"}"); s->close(restbed::BAD_REQUEST, response); } else { restbed::Bytes buf(b); @@ -280,11 +238,16 @@ DhtProxyServer::put(const std::shared_ptr& session) const // Build the Value from json auto value = std::make_shared(root); - Json::FastWriter writer; - dht_->put(infoHash, value); - s->close(restbed::OK, writer.write(value->toJson())); + dht_->put(infoHash, value, [s, value](bool ok) { + if (ok) { + Json::FastWriter writer; + s->close(restbed::OK, writer.write(value->toJson())); + } else { + s->close(restbed::BAD_GATEWAY, "{\"err\":\"put failed\"}"); + } + }); } else { - s->close(restbed::BAD_REQUEST, "Incorrect JSON"); + s->close(restbed::BAD_REQUEST, "{\"err\":\"Incorrect JSON\"}"); } } } else { @@ -310,7 +273,7 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons { if (dht_) { if(b.empty()) { - std::string response("Missing parameters"); + std::string response("{\"err\":\"Missing parameters\"}"); s->close(restbed::BAD_REQUEST, response); } else { restbed::Bytes buf(b); @@ -325,7 +288,7 @@ DhtProxyServer::putSigned(const std::shared_ptr& session) cons dht_->putSigned(infoHash, value); s->close(restbed::OK, writer.write(value->toJson())); } else { - s->close(restbed::BAD_REQUEST, "Incorrect JSON" + strJson); + s->close(restbed::BAD_REQUEST, "{\"err\":\"Incorrect JSON\"}"); } } } else { @@ -341,16 +304,16 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c const auto request = session->get_request(); int content_length = std::stoi(request->get_header("Content-Length", "0")); auto hash = request->get_path_parameter("hash"); - InfoHash infoHash(hash); - if (!infoHash) - infoHash = InfoHash::get(hash); + InfoHash key(hash); + if (!key) + key = InfoHash::get(hash); session->fetch(content_length, [=](const std::shared_ptr s, const restbed::Bytes& b) { if (dht_) { if(b.empty()) { - std::string response("Missing parameters"); + std::string response("{\"err\":\"Missing parameters\"}"); s->close(restbed::BAD_REQUEST, response); } else { restbed::Bytes buf(b); @@ -358,22 +321,17 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c Json::Reader reader; std::string strJson(buf.begin(), buf.end()); bool parsingSuccessful = reader.parse(strJson.c_str(), root); - auto toHash = root["to"].asString(); - if (parsingSuccessful && !toHash.empty()) { + InfoHash to(root["to"].asString()); + if (parsingSuccessful && toInfoHash) { auto value = std::make_shared(root); - auto toHash = request->get_path_parameter("to"); - InfoHash toInfoHash(toHash); - if (toInfoHash) - toInfoHash = InfoHash::get(toHash); - Json::FastWriter writer; - dht_->putEncrypted(infoHash, toInfoHash, value); + dht_->putEncrypted(key, to, value); s->close(restbed::OK, writer.write(value->toJson())); } else { if(!parsingSuccessful) - s->close(restbed::BAD_REQUEST, "Incorrect JSON"); + s->close(restbed::BAD_REQUEST, "{\"err\":\"Incorrect JSON\"}"); else - s->close(restbed::BAD_REQUEST, "No destination found"); + s->close(restbed::BAD_REQUEST, "{\"err\":\"No destination found\"}"); } } } else { From 65e88f2fa6f793fd6872c84fd5a6c709ffbe7601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blin?= Date: Wed, 8 Nov 2017 15:07:04 -0500 Subject: [PATCH 093/156] build: toInfoHash->to --- src/dht_proxy_server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dht_proxy_server.cpp b/src/dht_proxy_server.cpp index 586a8b77..6aef30ac 100644 --- a/src/dht_proxy_server.cpp +++ b/src/dht_proxy_server.cpp @@ -322,7 +322,7 @@ DhtProxyServer::putEncrypted(const std::shared_ptr& session) c std::string strJson(buf.begin(), buf.end()); bool parsingSuccessful = reader.parse(strJson.c_str(), root); InfoHash to(root["to"].asString()); - if (parsingSuccessful && toInfoHash) { + if (parsingSuccessful && to) { auto value = std::make_shared(root); Json::FastWriter writer; dht_->putEncrypted(key, to, value); From 5a049f89f3b87fb8957b28b2ba0b7aab5757a74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 8 Nov 2017 14:43:38 -0500 Subject: [PATCH 094/156] cmake: find Jsoncpp using pkgconfig --- CMakeLists.txt | 6 +++++- cmake/FindJsoncpp.cmake | 18 ------------------ 2 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 cmake/FindJsoncpp.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4efc5af5..8d909a10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,7 +46,7 @@ if (NOT OPENDHT_ARGON2) endif () if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT) find_package(Restbed REQUIRED) - find_package(Jsoncpp REQUIRED) + pkg_search_module(Jsoncpp jsoncpp) endif() # Build flags @@ -84,10 +84,14 @@ endif () if (Nettle_INCLUDE_DIRS) include_directories (SYSTEM "${Nettle_INCLUDE_DIRS}") endif () +if (RESTBED_INCLUDE) + include_directories (SYSTEM "${RESTBED_INCLUDE}") +endif () if (Jsoncpp_INCLUDE_DIRS) include_directories (SYSTEM "${Jsoncpp_INCLUDE_DIRS}") endif () link_directories (${Nettle_LIBRARY_DIRS}) +link_directories (${Jsoncpp_LIBRARY_DIRS}) include_directories ( ./ include/ diff --git a/cmake/FindJsoncpp.cmake b/cmake/FindJsoncpp.cmake deleted file mode 100644 index fccc2da3..00000000 --- a/cmake/FindJsoncpp.cmake +++ /dev/null @@ -1,18 +0,0 @@ -find_path (JSONCPP_INCLUDE jsoncpp - HINTS - "/usr/include" - "/usr/local/include" - "/opt/local/include" -) - -if (JSONCPP_INCLUDE) - message(STATUS "${green}Found Jsoncpp: ${JSONCPP_INCLUDE}") -else() - message(FATAL_ERROR "${red}Failed to locate Jsoncpp.}") -endif() - -if (JSONCPP_INCLUDE) - set(JSONCPP_FOUND TRUE) - set(Jsoncpp_LIBRARIES jsoncpp) - set(Jsoncpp_INCLUDE_DIRS ${JSONCPP_INCLUDE}/jsoncpp) -endif() From d7aaa4ae5617f9f3bf8f1012c56854841cd3eaae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 8 Nov 2017 15:05:52 -0500 Subject: [PATCH 095/156] proxy server: add Access-Control-Allow-Origin --- src/dht_proxy_server.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dht_proxy_server.cpp b/src/dht_proxy_server.cpp index 6aef30ac..de1f03c4 100644 --- a/src/dht_proxy_server.cpp +++ b/src/dht_proxy_server.cpp @@ -64,6 +64,7 @@ DhtProxyServer::DhtProxyServer(std::shared_ptr dht, in_port_t port) auto settings = std::make_shared(); settings->set_default_header("Content-Type", "application/json"); settings->set_default_header("Connection", "keep-alive"); + settings->set_default_header("Access-Control-Allow-Origin", "*"); std::chrono::milliseconds timeout(std::numeric_limits::max()); settings->set_connection_timeout(timeout); // there is a timeout, but really huge settings->set_port(port); From 7d7410b472bceca736c22b50275c9fee3ec3c674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 8 Nov 2017 15:10:23 -0500 Subject: [PATCH 096/156] dhtnode: allow to use arbitrary string as key Key will be h(key). --- tools/dhtnode.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index efad9767..b0bc35d9 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -201,6 +201,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl; continue; } + dht::InfoHash id; if (op == "il" or op == "ii") { // Pht syntax @@ -227,12 +228,17 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) else { // Dht syntax iss >> idstr; - InfoHash h {idstr}; - if (not isInfoHash(h)) - continue; + id = dht::InfoHash(idstr); + if (not id) { + if (idstr.empty()) { + std::cerr << "Syntax error: invalid InfoHash." << std::endl; + continue; + } + id = InfoHash::get(idstr); + std::cout << "Using h(" << idstr << ") = " << id << std::endl; + } } - dht::InfoHash id {idstr}; // Dht auto start = std::chrono::high_resolution_clock::now(); if (op == "g") { From b549055034a96b990e03c159c28d8bc82d7368a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Mon, 6 Nov 2017 15:47:05 -0500 Subject: [PATCH 097/156] DhtRunner: use SockAddr --- include/opendht/dhtrunner.h | 8 +-- src/dhtrunner.cpp | 103 +++++++++++++++++++----------------- 2 files changed, 57 insertions(+), 54 deletions(-) diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index fd3346c0..fb78f1af 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -202,7 +202,7 @@ public: * Insert known nodes to the routing table, without necessarly ping them. * Usefull to restart a node and get things running fast without putting load on the network. */ - void bootstrap(const std::vector>& nodes, DoneCallbackSimple&& cb={}); + void bootstrap(const std::vector& nodes, DoneCallbackSimple&& cb={}); void bootstrap(const SockAddr& addr, DoneCallbackSimple&& cb={}); /** @@ -329,7 +329,7 @@ public: * @param threaded: If false, loop() must be called periodically. Otherwise a thread is launched. * @param cb: Optional callback to receive general state information. */ - void run(const sockaddr_in* local4, const sockaddr_in6* local6, Config config); + void run(const SockAddr& local4, const SockAddr& local6, Config config); /** * Same as @run(sockaddr_in, sockaddr_in6, Identity, bool, StatusCallback), but with string IP addresses and service (port). @@ -373,10 +373,10 @@ private: */ void tryBootstrapContinuously(); - void doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDhtConfig config); + void doRun(const SockAddr& sin4, const SockAddr& sin6, SecureDhtConfig config); time_point loop_(); - static std::vector> getAddrInfo(const std::string& host, const std::string& service); + static std::vector getAddrInfo(const std::string& host, const std::string& service); NodeStatus getStatus() const { return std::max(status4, status6); diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index 66b4ebbd..e2850580 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -58,15 +58,13 @@ DhtRunner::~DhtRunner() void DhtRunner::run(in_port_t port, DhtRunner::Config config) { - sockaddr_in sin4; - std::fill_n((uint8_t*)&sin4, sizeof(sin4), 0); - sin4.sin_family = AF_INET; - sin4.sin_port = htons(port); - sockaddr_in6 sin6; - std::fill_n((uint8_t*)&sin6, sizeof(sin6), 0); - sin6.sin6_family = AF_INET6; - sin6.sin6_port = htons(port); - run(&sin4, &sin6, config); + SockAddr sin4; + sin4.setFamily(AF_INET); + sin4.setPort(port); + SockAddr sin6; + sin6.setFamily(AF_INET6); + sin6.setPort(port); + run(sin4, sin6, config); } void @@ -74,12 +72,12 @@ DhtRunner::run(const char* ip4, const char* ip6, const char* service, DhtRunner: { auto res4 = getAddrInfo(ip4, service); auto res6 = getAddrInfo(ip6, service); - run(res4.empty() ? nullptr : (sockaddr_in*) &res4.front().first, - res6.empty() ? nullptr : (sockaddr_in6*)&res6.front().first, config); + run(res4.empty() ? SockAddr() : res4.front(), + res6.empty() ? SockAddr() : res6.front(), config); } void -DhtRunner::run(const sockaddr_in* local4, const sockaddr_in6* local6, DhtRunner::Config config) +DhtRunner::run(const SockAddr& local4, const SockAddr& local6, DhtRunner::Config config) { if (running) return; @@ -375,8 +373,34 @@ DhtRunner::loop_() return wakeup; } + +int bindSocket(const SockAddr& addr, SockAddr& bound) +{ + bool is_ipv6 = addr.getFamily() == AF_INET6; + int sock = socket(is_ipv6 ? PF_INET6 : PF_INET, SOCK_DGRAM, 0); + if (sock >= 0) { + int set = 1; +#ifdef SO_NOSIGPIPE + setsockopt(sock, SOL_SOCKET, SO_NOSIGPIPE, (void *)&set, sizeof(set)); +#endif + if (is_ipv6) + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void *)&set, sizeof(set)); + int rc = bind(sock, addr.get(), addr.getLength()); + if(rc < 0) + throw DhtException("Can't bind socket on " + addr.toString() + " " + strerror(rc)); + sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + getsockname(sock, (sockaddr*)&ss, &ss_len); + bound = {ss, ss_len}; + return sock; + } else { + throw DhtException(std::string("Can't open socket: ") + strerror(sock)); + } + return -1; +} + void -DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::Config config) +DhtRunner::doRun(const SockAddr& sin4, const SockAddr& sin6, SecureDht::Config config) { dht_.reset(); @@ -384,40 +408,19 @@ DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::C s6 = -1; bound4 = {}; - if (sin4) { - s4 = socket(PF_INET, SOCK_DGRAM, 0); - if(s4 >= 0) { - int rc = bind(s4, (sockaddr*)sin4, sizeof(sockaddr_in)); - if(rc < 0) - throw DhtException("Can't bind IPv4 socket on " + dht::print_addr((sockaddr*)sin4, sizeof(sockaddr_in))); - sockaddr_storage ss; - socklen_t ss_len = sizeof(ss); - getsockname(s4, (sockaddr*)&ss, &ss_len); - bound4 = {ss, ss_len}; - } - } + if (sin4) + s4 = bindSocket(sin4, bound4); #if 1 bound6 = {}; - if (sin6) { - s6 = socket(PF_INET6, SOCK_DGRAM, 0); - if(s6 >= 0) { - int val = 1; - int rc = setsockopt(s6, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&val, sizeof(val)); - if(rc < 0) - throw DhtException("Can't set IPV6_V6ONLY"); - - rc = bind(s6, (sockaddr*)sin6, sizeof(sockaddr_in6)); - if(rc < 0) - throw DhtException("Can't bind IPv6 socket on " + dht::print_addr((sockaddr*)sin6, sizeof(sockaddr_in6))); - sockaddr_storage ss; - socklen_t ss_len = sizeof(ss); - getsockname(s6, (sockaddr*)&ss, &ss_len); - bound6 = {ss, ss_len}; - } - } + if (sin6) + s6 = bindSocket(sin6, bound6); #endif + std::cerr << "s4: " << s4 << " bound on " << bound4.toString() << std::endl; + std::cerr << "s6: " << s6 << " bound on " << bound6.toString() << std::endl; + + dht_ = std::unique_ptr(new SecureDht {s4, s6, config}); rcv_thread = std::thread([this,s4,s6]() { @@ -490,7 +493,8 @@ DhtRunner::get(const std::string& key, GetCallback vcb, DoneCallbackSimple dcb, { get(InfoHash::get(key), vcb, dcb, f, w); } -void DhtRunner::query(const InfoHash& hash, QueryCallback cb, DoneCallback done_cb, Query q) { +void +DhtRunner::query(const InfoHash& hash, QueryCallback cb, DoneCallback done_cb, Query q) { { std::lock_guard lck(storage_mtx); pending_ops.emplace([=](SecureDht& dht) mutable { @@ -695,10 +699,10 @@ DhtRunner::tryBootstrapContinuously() }); } -std::vector> +std::vector DhtRunner::getAddrInfo(const std::string& host, const std::string& service) { - std::vector> ips {}; + std::vector ips {}; if (host.empty()) return ips; @@ -712,8 +716,7 @@ DhtRunner::getAddrInfo(const std::string& host, const std::string& service) addrinfo* infop = info; while (infop) { - ips.emplace_back(sockaddr_storage(), infop->ai_addrlen); - std::copy_n((uint8_t*)infop->ai_addr, infop->ai_addrlen, (uint8_t*)&ips.back().first); + ips.emplace_back(infop->ai_addr, infop->ai_addrlen); infop = infop->ai_next; } freeaddrinfo(info); @@ -737,13 +740,13 @@ DhtRunner::clearBootstrap() } void -DhtRunner::bootstrap(const std::vector>& nodes, DoneCallbackSimple&& cb) +DhtRunner::bootstrap(const std::vector& nodes, DoneCallbackSimple&& cb) { std::lock_guard lck(storage_mtx); pending_ops_prio.emplace([=](SecureDht& dht) mutable { auto rem = cb ? std::make_shared>(nodes.size(), false) : nullptr; - for (auto& node : nodes) - dht.pingNode((sockaddr*)&node.first, node.second, cb ? [rem,cb](bool ok) { + for (const auto& node : nodes) + dht.pingNode(node.get(), node.getLength(), cb ? [rem,cb](bool ok) { auto& r = *rem; r.first--; r.second |= ok; From 113bebabd4bde20f830b117c61fb4025d96737b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 8 Nov 2017 15:14:01 -0500 Subject: [PATCH 098/156] DhtRunner: remove log --- src/dhtrunner.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index e2850580..dc7c7ede 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -417,10 +417,6 @@ DhtRunner::doRun(const SockAddr& sin4, const SockAddr& sin6, SecureDht::Config c s6 = bindSocket(sin6, bound6); #endif - std::cerr << "s4: " << s4 << " bound on " << bound4.toString() << std::endl; - std::cerr << "s6: " << s6 << " bound on " << bound6.toString() << std::endl; - - dht_ = std::unique_ptr(new SecureDht {s4, s6, config}); rcv_thread = std::thread([this,s4,s6]() { From 153641dde5a3dd91736ab08dd03d209822edb1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 8 Nov 2017 15:19:33 -0500 Subject: [PATCH 099/156] tools: add JavaScript proxy node --- tools/proxy_node.html | 117 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tools/proxy_node.html diff --git a/tools/proxy_node.html b/tools/proxy_node.html new file mode 100644 index 00000000..4f60a7bf --- /dev/null +++ b/tools/proxy_node.html @@ -0,0 +1,117 @@ + + + + + + OpenDHT tester + + + + + + + +
+

OpenDHT tester

+
+
+
+
+
+
+
+
+ + + + +
+
+
+
+
+
+ + + + +
+
+
+
+
+
+
+
+
+ + From 111ac57b6ff327be504d31104e5febe704b03c27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Thu, 9 Nov 2017 12:23:30 -0500 Subject: [PATCH 100/156] tools: JS node: add listen support --- tools/proxy_node.html | 56 +++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/tools/proxy_node.html b/tools/proxy_node.html index 4f60a7bf..f7929ce2 100644 --- a/tools/proxy_node.html +++ b/tools/proxy_node.html @@ -16,19 +16,37 @@ const valueGetElement = function(o) { return d; }; $(function() { + var request = undefined; + const getTools = $("#getTools"); const getBtn = $("#getBtn"); + const getDropdown = $("#getDropdown"); + const listenBtn = $("#listenBtn").click(function(){onGet('LISTEN');}); + const getStopBtn = $("#getStopBtn").detach().click(function(){ + if (request === undefined) + return; + request.abort(); + getStopBtn.detach(); + getDropdown.show(); + request = undefined; + }); const putBtn = $("#putBtn"); const result = $("#dhtResult"); const group = $('
    ').appendTo(result); - onGet = function ( ) { + onGet = function (method) { + if (request !== undefined) + return false; const input = $("#getKey").val(); - const req = new XMLHttpRequest(); - req.onreadystatechange = function(event) { + request = new XMLHttpRequest(); + request.onreadystatechange = function(event) { if (this.readyState >= XMLHttpRequest.LOADING) { - if (this.readyState == XMLHttpRequest.DONE) + if (this.readyState == XMLHttpRequest.DONE) { getBtn.button('reset'); - group.empty(); + getStopBtn.detach(); + getDropdown.show(); + request = undefined; + } if (this.status === 200) { + group.empty(); const elements = this.responseText.split("\n"); elements.forEach(function(element) { if (!element || element.length == 0) @@ -37,18 +55,23 @@ $(function() { const d = window.atob(o.data); $('
  • ').append(valueGetElement(o)).appendTo(group); }); - } else { + } else if (this.status !== 0) { group.empty().append($('
  • ').text("Error loading content: " + this.statusText)); } } }; - req.onerror = function(event) { + request.onerror = function(event) { getBtn.button('reset'); + getStopBtn.detach(); + getDropdown.show(); group.empty().append($('
  • ').text("Error loading content.")); + request = undefined; }; - req.open('GET', "http://127.0.0.1:8080/" + input, true); - req.send(null); + request.open(method, "http://127.0.0.1:8080/" + input, true); + request.send(null); getBtn.button('loading'); + getStopBtn.appendTo(getTools); + getDropdown.hide(); return false; }; @@ -65,7 +88,7 @@ $(function() { dataType: 'json', success: function( result ) { putBtn.button('reset'); - $('
  • ').append(valueGetElement(result)).appendTo(group.empty()); + //$('
  • ').append(valueGetElement(result)).appendTo(group.empty()); }, error: function(result) { putBtn.button('reset'); @@ -88,11 +111,18 @@ $(function() {
    -
    +
    - - + + + + + +
    From 607e5eeccfbe6a49b7f9fd9e86dde9860cc0af5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 10 Nov 2017 17:28:26 -0500 Subject: [PATCH 101/156] tools: use https url --- tools/dhtchat.cpp | 2 +- tools/dhtnode.cpp | 2 +- tools/dhtscanner.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/dhtchat.cpp b/tools/dhtchat.cpp index 3bd89f1c..34ae65e9 100644 --- a/tools/dhtchat.cpp +++ b/tools/dhtchat.cpp @@ -45,7 +45,7 @@ void print_node_info(const DhtRunner& dht, const dht_params&) { void print_usage() { std::cout << "Usage: dhtchat [-n network_id] [-p local_port] [-b bootstrap_host[:port]]" << std::endl << std::endl; std::cout << "dhtchat, a simple OpenDHT command line chat client." << std::endl; - std::cout << "Report bugs to: http://opendht.net" << std::endl; + std::cout << "Report bugs to: https://opendht.net" << std::endl; } int diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index b0bc35d9..8b7bb75f 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -31,7 +31,7 @@ using namespace dht; void print_usage() { std::cout << "Usage: dhtnode [-v [-l logfile]] [-i] [-d] [-n network_id] [-p local_port] [-b bootstrap_host[:port]] [--proxyserver local_port]" << std::endl << std::endl; std::cout << "dhtnode, a simple OpenDHT command line node runner." << std::endl; - std::cout << "Report bugs to: http://opendht.net" << std::endl; + std::cout << "Report bugs to: https://opendht.net" << std::endl; } void print_id_req() { diff --git a/tools/dhtscanner.cpp b/tools/dhtscanner.cpp index 137378c0..46baf597 100644 --- a/tools/dhtscanner.cpp +++ b/tools/dhtscanner.cpp @@ -32,7 +32,7 @@ using namespace dht; void print_usage() { std::cout << "Usage: dhtscanner [-n network_id] [-p local_port] [-b bootstrap_host[:port]]" << std::endl << std::endl; std::cout << "dhtscanner, a simple OpenDHT command line utility generating scan result the network." << std::endl; - std::cout << "Report bugs to: http://opendht.net" << std::endl; + std::cout << "Report bugs to: https://opendht.net" << std::endl; } struct snode_compare { From 9948079c9749c32f9e1c5a43ab68c9172baa065c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 10 Nov 2017 17:47:09 -0500 Subject: [PATCH 102/156] SockAddr: add resolve() --- include/opendht/sockaddr.h | 3 +++ src/utils.cpp | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/include/opendht/sockaddr.h b/include/opendht/sockaddr.h index 8f946325..37efa2d9 100644 --- a/include/opendht/sockaddr.h +++ b/include/opendht/sockaddr.h @@ -36,6 +36,7 @@ typedef uint16_t in_port_t; #include #include +#include #include #include @@ -71,6 +72,8 @@ public: */ SockAddr(const sockaddr_storage& ss, socklen_t len) : SockAddr((const sockaddr*)&ss, len) {} + static std::vector resolve(const std::string& host, const std::string& service = {}); + bool operator<(const SockAddr& o) const { if (len != o.len) return len < o.len; diff --git a/src/utils.cpp b/src/utils.cpp index 87826e1f..18e946a0 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -27,6 +27,31 @@ namespace dht { +std::vector +SockAddr::resolve(const std::string& host, const std::string& service) +{ + std::vector ips {}; + if (host.empty()) + return ips; + + addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_socktype = SOCK_DGRAM; + addrinfo* info = nullptr; + int rc = getaddrinfo(host.c_str(), service.c_str(), &hints, &info); + if(rc != 0) + throw std::invalid_argument(std::string("Error: `") + host + ":" + service + "`: " + gai_strerror(rc)); + + addrinfo* infop = info; + while (infop) { + ips.emplace_back(infop->ai_addr, infop->ai_addrlen); + infop = infop->ai_next; + } + freeaddrinfo(info); + return ips; +} + + std::string print_addr(const sockaddr* sa, socklen_t slen) { From 161769f4ece71d5f2f8dd2d14651c8a5d6d11948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 10 Nov 2017 17:48:13 -0500 Subject: [PATCH 103/156] SockAddr: use host endianess for bitwise-shift operations --- src/utils.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/utils.cpp b/src/utils.cpp index 18e946a0..9cd9f492 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -94,7 +94,8 @@ SockAddr::isLoopback() const { switch (getFamily()) { case AF_INET: { - uint8_t b1 = (uint8_t)(getIPv4().sin_addr.s_addr >> 24); + auto addr_host = ntohl(getIPv4().sin_addr.s_addr); + uint8_t b1 = (uint8_t)(addr_host >> 24); return b1 == 127; } case AF_INET6: @@ -111,10 +112,11 @@ SockAddr::isPrivate() const return true; } switch (getFamily()) { - case AF_INET: + case AF_INET: { + auto addr_host = ntohl(getIPv4().sin_addr.s_addr); uint8_t b1, b2; - b1 = (uint8_t)(getIPv4().sin_addr.s_addr >> 24); - b2 = (uint8_t)((getIPv4().sin_addr.s_addr >> 16) & 0x0ff); + b1 = (uint8_t)(addr_host >> 24); + b2 = (uint8_t)((addr_host >> 16) & 0x0ff); // 10.x.y.z if (b1 == 10) return true; @@ -125,6 +127,7 @@ SockAddr::isPrivate() const if ((b1 == 192) && (b2 == 168)) return true; return false; + } case AF_INET6: { const uint8_t* addr6 = reinterpret_cast(&getIPv6().sin6_addr); if (addr6[0] == 0xfc) From 747f86f8f4348d022cae47b6ef5e9dc28d10bfa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Fri, 10 Nov 2017 17:59:05 -0500 Subject: [PATCH 104/156] dhtrunner: use SockAddr::resolve --- include/opendht/dhtrunner.h | 2 -- src/dhtrunner.cpp | 30 +++--------------------------- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index fb78f1af..c95c2eb8 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -376,8 +376,6 @@ private: void doRun(const SockAddr& sin4, const SockAddr& sin6, SecureDhtConfig config); time_point loop_(); - static std::vector getAddrInfo(const std::string& host, const std::string& service); - NodeStatus getStatus() const { return std::max(status4, status6); } diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index dc7c7ede..d9816200 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -70,8 +70,8 @@ DhtRunner::run(in_port_t port, DhtRunner::Config config) void DhtRunner::run(const char* ip4, const char* ip6, const char* service, DhtRunner::Config config) { - auto res4 = getAddrInfo(ip4, service); - auto res6 = getAddrInfo(ip6, service); + auto res4 = SockAddr::resolve(ip4, service); + auto res6 = SockAddr::resolve(ip6, service); run(res4.empty() ? SockAddr() : res4.front(), res6.empty() ? SockAddr() : res6.front(), config); } @@ -664,7 +664,7 @@ DhtRunner::tryBootstrapContinuously() for (auto it = nodes.rbegin(); it != nodes.rend(); it++) { ++ping_count; try { - bootstrap(getAddrInfo(it->first, it->second), [&](bool) { + bootstrap(SockAddr::resolve(it->first, it->second), [&](bool) { if (not running) return; { @@ -695,30 +695,6 @@ DhtRunner::tryBootstrapContinuously() }); } -std::vector -DhtRunner::getAddrInfo(const std::string& host, const std::string& service) -{ - std::vector ips {}; - if (host.empty()) - return ips; - - addrinfo hints; - memset(&hints, 0, sizeof(hints)); - hints.ai_socktype = SOCK_DGRAM; - addrinfo* info = nullptr; - int rc = getaddrinfo(host.c_str(), service.c_str(), &hints, &info); - if(rc != 0) - throw std::invalid_argument(std::string("Error: `") + host + ":" + service + "`: " + gai_strerror(rc)); - - addrinfo* infop = info; - while (infop) { - ips.emplace_back(infop->ai_addr, infop->ai_addrlen); - infop = infop->ai_next; - } - freeaddrinfo(info); - return ips; -} - void DhtRunner::bootstrap(const std::string& host, const std::string& service) { From 72b556a09b9ffe79f92f5228cfe64f991306d8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 14 Nov 2017 19:38:55 +0100 Subject: [PATCH 105/156] tools: JS node: cleanup --- tools/proxy_node.html | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/proxy_node.html b/tools/proxy_node.html index f7929ce2..5661c18f 100644 --- a/tools/proxy_node.html +++ b/tools/proxy_node.html @@ -21,14 +21,21 @@ $(function() { const getBtn = $("#getBtn"); const getDropdown = $("#getDropdown"); const listenBtn = $("#listenBtn").click(function(){onGet('LISTEN');}); - const getStopBtn = $("#getStopBtn").detach().click(function(){ + const setGetRequest = function() { + getBtn.button('loading'); + getStopBtn.appendTo(getTools); + getDropdown.hide(); + } + const clearGetRequest = function() { if (request === undefined) return; request.abort(); + request = undefined; getStopBtn.detach(); getDropdown.show(); - request = undefined; - }); + getBtn.button('reset'); + } + const getStopBtn = $("#getStopBtn").detach().click(clearGetRequest); const putBtn = $("#putBtn"); const result = $("#dhtResult"); const group = $('
      ').appendTo(result); @@ -36,14 +43,12 @@ $(function() { if (request !== undefined) return false; const input = $("#getKey").val(); + group.empty(); request = new XMLHttpRequest(); request.onreadystatechange = function(event) { if (this.readyState >= XMLHttpRequest.LOADING) { if (this.readyState == XMLHttpRequest.DONE) { - getBtn.button('reset'); - getStopBtn.detach(); - getDropdown.show(); - request = undefined; + clearGetRequest(); } if (this.status === 200) { group.empty(); @@ -61,17 +66,12 @@ $(function() { } }; request.onerror = function(event) { - getBtn.button('reset'); - getStopBtn.detach(); - getDropdown.show(); + clearGetRequest(); group.empty().append($('
    • ').text("Error loading content.")); - request = undefined; }; request.open(method, "http://127.0.0.1:8080/" + input, true); request.send(null); - getBtn.button('loading'); - getStopBtn.appendTo(getTools); - getDropdown.hide(); + setGetRequest(); return false; }; From 72d790953a3a049f1798a7d38e6c296903355d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 14 Nov 2017 20:58:18 +0100 Subject: [PATCH 106/156] tools: JS node: allow to set the proxy address --- tools/proxy_node.html | 57 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/tools/proxy_node.html b/tools/proxy_node.html index 5661c18f..e221a55b 100644 --- a/tools/proxy_node.html +++ b/tools/proxy_node.html @@ -11,12 +11,14 @@ -
      -

      OpenDHT tester

      -
      -
      -
      +
      +
      +
      +
      +

      OpenDHT tester

      +
      +
      +
      +
      +
      + + + + +
      +
      +
      +
      +
      +
      +
      +
      -
      +
      - @@ -127,7 +162,7 @@ $(function() {
      -
      +
      @@ -137,11 +172,9 @@ $(function() {
      -
      -
      From 93498ecf0b00ef83c2cda9eb39a6909ebed082f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Tue, 14 Nov 2017 21:43:44 +0100 Subject: [PATCH 107/156] tools: JS node: cleanup --- tools/proxy_node.html | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tools/proxy_node.html b/tools/proxy_node.html index e221a55b..735f0ed7 100644 --- a/tools/proxy_node.html +++ b/tools/proxy_node.html @@ -9,16 +9,16 @@