uv

* improved Doxygen documentation
* cmake: add support to build documentation
* cmake: improve build on macOS
* API change: storage policies, getPublicAddress now use SockAddr
* NodeExport: added msgpack serialization
* fixed issue that could cause occasional request failures
* minor bug fixes
* minor performance improvements

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required (VERSION 3.1)
 project (opendht)
 set (opendht_VERSION_MAJOR 1)
-set (opendht_VERSION_MINOR 3.6)
+set (opendht_VERSION_MINOR 4.0)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
 set (VERSION "${opendht_VERSION}")
@@ -15,9 +15,14 @@ option (OPENDHT_TOOLS "Build DHT tools" ON)
 option (OPENDHT_SYSTEMD "Install systemd module" OFF)
 option (OPENDHT_ARGON2 "Use included argon2 sources" OFF)
 option (OPENDHT_LTO "Build with LTO" OFF)
+option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF)
+
+find_package(Doxygen)
+option (OPENDHT_DOCUMENTATION "Create and install the HTML based API documentation (requires Doxygen)" ${DOXYGEN_FOUND})
 
 # Dependencies
 list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
+find_package (Threads)
 find_package (PkgConfig)
 pkg_search_module(libuv libuv)
 pkg_search_module(libsodium libsodium)
@@ -41,7 +46,10 @@ endif ()
 # Build flags
 set (CMAKE_CXX_STANDARD 14)
 set (CMAKE_CXX_STANDARD_REQUIRED on)
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden")
+if (OPENDHT_SANITIZE)
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fstack-protector-strong")
+endif ()
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMSGPACK_DISABLE_LEGACY_NIL -DMSGPACK_DISABLE_LEGACY_CONVERT")
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
@@ -70,7 +78,13 @@ endif ()
 if (Nettle_INCLUDE_DIRS)
     include_directories (SYSTEM "${Nettle_INCLUDE_DIRS}")
 endif ()
-link_directories (${Nettle_LIBRARY_DIRS})
+if (libuv_INCLUDE_DIRS)
+    include_directories (SYSTEM "${libuv_INCLUDE_DIRS}")
+endif ()
+if (libsodium_INCLUDE_DIRS)
+    include_directories (SYSTEM "${libsodium_INCLUDE_DIRS}")
+endif ()
+link_directories (${libuv_LIBRARY_DIRS} ${libsodium_LIBRARY_DIRS} ${Nettle_LIBRARY_DIRS})
 include_directories (
     ./
     include/
@@ -86,6 +100,7 @@ set (libdir "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
 set (includedir "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
 set (bindir "${CMAKE_INSTALL_FULL_BINDIR}")
 set (sysconfdir "${CMAKE_INSTALL_FULL_SYSCONFDIR}")
+set (top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}")
 
 # Sources
 list (APPEND opendht_SOURCES
@@ -99,6 +114,7 @@ list (APPEND opendht_SOURCES
     src/storage.h
     src/listener.h
     src/search.h
+    src/parsed_message.h
     src/request.h
     src/callbacks.cpp
     src/routing_table.cpp
@@ -168,7 +184,7 @@ if (OPENDHT_STATIC)
         target_link_libraries(opendht-static ${argon2_LIBRARIES})
         target_include_directories(opendht-static SYSTEM PRIVATE ${argon2_INCLUDE_DIRS})
     endif ()
-    target_link_libraries(opendht-static ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${libuv_LIBRARIES} ${libsodium_LIBRARIES})
+    target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${libuv_LIBRARIES} ${libsodium_LIBRARIES})
     install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht)
 endif ()
 
@@ -181,15 +197,15 @@ if (OPENDHT_SHARED)
     set_target_properties (opendht PROPERTIES SOVERSION ${opendht_VERSION_MAJOR} VERSION ${opendht_VERSION})
     target_compile_definitions(opendht PRIVATE OPENDHT_BUILD)
     if (OPENDHT_LTO)
-        target_link_libraries(opendht -flto -fuse-linker-plugin)
+        target_link_libraries(opendht PRIVATE -flto -fuse-linker-plugin)
     endif ()
     if (OPENDHT_ARGON2)
         target_include_directories(opendht SYSTEM PRIVATE argon2)
     else ()
-        target_link_libraries(opendht ${argon2_LIBRARIES})
+        target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES})
         target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS})
     endif ()
-    target_link_libraries(opendht ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${libuv_LIBRARIES} ${libsodium_LIBRARIES})
+    target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${libuv_LIBRARIES} ${libsodium_LIBRARIES})
     install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht)
 endif ()
 

configure.ac

@@ -1,7 +1,7 @@
 dnl define macros
 m4_define([opendht_major_version], 1)
-m4_define([opendht_minor_version], 3)
-m4_define([opendht_patch_version], 6)
+m4_define([opendht_minor_version], 4)
+m4_define([opendht_patch_version], 0)
 m4_define([opendht_version],
 		  [opendht_major_version.opendht_minor_version.opendht_patch_version])
 

doc/CMakeLists.txt

@@ -1,3 +1,16 @@
 if (OPENDHT_TOOLS)
 	INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/dhtnode.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1)
 endif ()
+
+if (OPENDHT_DOCUMENTATION)
+    if (NOT DOXYGEN_FOUND)
+         message(FATAL_ERROR "Doxygen is needed to build the documentation.")
+    endif()
+    configure_file (Doxyfile.in Doxyfile @ONLY)
+    add_custom_target(doc ALL
+                      COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
+                      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                      COMMENT "Generating API documentation with Doxygen"
+                      VERBATIM)
+    install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html DESTINATION share/doc/opendht)
+endif()

include/opendht/crypto.h

@@ -40,6 +40,10 @@ extern "C" {
 #include <sys/mman.h>
 
 namespace dht {
+
+/**
+ * Contains all crypto primitives
+ */
 namespace crypto {
 
 class OPENDHT_PUBLIC CryptoException : public std::runtime_error {

include/opendht/default_types.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include "value.h"
+#include "sockaddr.h"
 
 namespace dht {
 enum class ImStatus : uint8_t {
@@ -45,7 +46,7 @@ public:
 
     static Value::Filter getFilter() { return {}; }
 
-    static bool storePolicy(InfoHash key, std::shared_ptr<Value>& value, InfoHash from, const sockaddr* from_addr, socklen_t from_len);
+    static bool storePolicy(InfoHash key, std::shared_ptr<Value>& value, const InfoHash& from, const SockAddr&);
 
     static Value::Filter ServiceFilter(std::string s);
 
@@ -204,16 +205,12 @@ private:
 public:
     static const ValueType TYPE;
 
-    IpServiceAnnouncement(in_port_t p = 0) {
-        ss.ss_family = 0;
-        setPort(p);
+    IpServiceAnnouncement(sa_family_t family = AF_UNSPEC, in_port_t p = 0) {
+        addr.setFamily(family);
+        addr.setPort(p);
     }
 
-    IpServiceAnnouncement(const sockaddr* sa, socklen_t sa_len) {
-        ss.ss_family = 0;
-        if (sa)
-            std::copy_n((const uint8_t*)sa, sa_len, (uint8_t*)&ss);
-    }
+    IpServiceAnnouncement(const SockAddr& sa) : addr(sa) {}
 
     IpServiceAnnouncement(const Blob& b) {
         msgpack_unpack(unpackMsg(b).get());
@@ -222,53 +219,40 @@ public:
     template <typename Packer>
     void msgpack_pack(Packer& pk) const
     {
-        pk.pack_array(2);
-        pk.pack(reinterpret_cast<const sockaddr_in*>(&ss)->sin_port);
-        if (ss.ss_family == AF_INET) {
-            pk.pack_bin(sizeof(in_addr));
-            pk.pack_bin_body((const char*)&reinterpret_cast<const sockaddr_in*>(&ss)->sin_addr, sizeof(in_addr));
-        } else if (ss.ss_family == AF_INET6) {
-            pk.pack_bin(sizeof(in6_addr));
-            pk.pack_bin_body((const char*)&reinterpret_cast<const sockaddr_in6*>(&ss)->sin6_addr, sizeof(in6_addr));
-        }
+        pk.pack_bin(addr.getLength());
+        pk.pack_bin_body((const char*)addr.get(), addr.getLength());
     }
 
     virtual void msgpack_unpack(msgpack::object o)
     {
-        if (o.type != msgpack::type::ARRAY) throw msgpack::type_error();
-        if (o.via.array.size < 2) throw msgpack::type_error();
-        reinterpret_cast<sockaddr_in*>(&ss)->sin_port = o.via.array.ptr[0].as<in_port_t>();
-        auto ip_dat = o.via.array.ptr[1].as<Blob>();
-        if (ip_dat.size() == sizeof(in_addr))
-            std::copy(ip_dat.begin(), ip_dat.end(), (char*)&reinterpret_cast<sockaddr_in*>(&ss)->sin_addr);
-        else if (ip_dat.size() == sizeof(in6_addr))
-            std::copy(ip_dat.begin(), ip_dat.end(), (char*)&reinterpret_cast<sockaddr_in6*>(&ss)->sin6_addr);
+        if (o.type == msgpack::type::BIN)
+            addr = {(sockaddr*)o.via.bin.ptr, (socklen_t)o.via.bin.size};
         else
             throw msgpack::type_error();
     }
 
     in_port_t getPort() const {
-        return ntohs(reinterpret_cast<const sockaddr_in*>(&ss)->sin_port);
+        return addr.getPort();
     }
     void setPort(in_port_t p) {
-        reinterpret_cast<sockaddr_in*>(&ss)->sin_port = htons(p);
+        addr.setPort(p);
     }
 
-    sockaddr_storage getPeerAddr() const {
-        return ss;
+    const SockAddr& getPeerAddr() const {
+        return addr;
     }
 
     virtual const ValueType& getType() const {
         return TYPE;
     }
 
-    static bool storePolicy(InfoHash, std::shared_ptr<Value>&, InfoHash, const sockaddr*, socklen_t);
+    static bool storePolicy(InfoHash, std::shared_ptr<Value>&, const InfoHash&, const SockAddr&);
 
     /** print value for debugging */
     friend std::ostream& operator<< (std::ostream&, const IpServiceAnnouncement&);
 
 private:
-    sockaddr_storage ss;
+    SockAddr addr;
 };
 
 

include/opendht/dht.h

@@ -168,8 +168,8 @@ public:
      * @param key the key for which to query data for.
      * @param cb a function called when new values are found on the network.
      *           It should return false to stop the operation.
-     * @param donecb a function called when the operation is complete.
-                     cb and donecb won't be called again afterward.
+     * @param done_cb a function called when the operation is complete.
+                     cb and done_cb won't be called again afterward.
      * @param q a query used to filter values on the remotes before they send a
      *          response.
      */
@@ -407,7 +407,6 @@ private:
     // timing
     Scheduler scheduler;
     Sp<Job> nextNodesConfirmation {};
-    time_point mybucket_grow_time {time_point::min()}, mybucket6_grow_time {time_point::min()};
 
     net::NetworkEngine network_engine;
     unsigned pending_pings4 {0};
@@ -431,7 +430,7 @@ private:
 
     // Storage
     void storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t tid, Query&& = {});
-    bool storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr* sa = nullptr);
+    bool storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr& sa = {});
     void expireStore();
     void expireStorage(InfoHash h);
     void expireStore(decltype(store)::iterator);
@@ -514,7 +513,7 @@ private:
      * @param query   The query sent to the node.
      */
     void searchNodeGetDone(const net::Request& status,
-            net::NetworkEngine::RequestAnswer&& answer,
+            net::RequestAnswer&& answer,
             std::weak_ptr<Search> ws,
             Sp<Query> query);
 
@@ -567,42 +566,42 @@ private:
     /* when our address is reported by a distant peer. */
     void onReportedAddr(const InfoHash& id, const SockAddr&);
     /* when we receive a ping request */
-    net::NetworkEngine::RequestAnswer onPing(Sp<Node> node);
+    net::RequestAnswer onPing(Sp<Node> node);
     /* when we receive a "find node" request */
-    net::NetworkEngine::RequestAnswer onFindNode(Sp<Node> node, const InfoHash& hash, want_t want);
+    net::RequestAnswer onFindNode(Sp<Node> node, const InfoHash& hash, want_t want);
     void onFindNodeDone(const Sp<Node>& status,
-            net::NetworkEngine::RequestAnswer& a,
+            net::RequestAnswer& a,
             Sp<Search> sr);
     /* when we receive a "get values" request */
-    net::NetworkEngine::RequestAnswer onGetValues(Sp<Node> node,
+    net::RequestAnswer onGetValues(Sp<Node> node,
             const InfoHash& hash,
             want_t want,
             const Query& q);
     void onGetValuesDone(const Sp<Node>& status,
-            net::NetworkEngine::RequestAnswer& a,
+            net::RequestAnswer& a,
             Sp<Search>& sr,
             const Sp<Query>& orig_query);
     /* when we receive a listen request */
-    net::NetworkEngine::RequestAnswer onListen(Sp<Node> node,
+    net::RequestAnswer onListen(Sp<Node> node,
             const InfoHash& hash,
             const Blob& token,
             size_t socket_id,
             const Query& query);
     void onListenDone(const Sp<Node>& status,
-            net::NetworkEngine::RequestAnswer& a,
+            net::RequestAnswer& a,
             Sp<Search>& sr);
     /* when we receive an announce request */
-    net::NetworkEngine::RequestAnswer onAnnounce(Sp<Node> node,
+    net::RequestAnswer onAnnounce(Sp<Node> node,
             const InfoHash& hash,
             const Blob& token,
             const std::vector<Sp<Value>>& v,
             const time_point& created);
-    net::NetworkEngine::RequestAnswer onRefresh(Sp<Node> node,
+    net::RequestAnswer onRefresh(Sp<Node> node,
             const InfoHash& hash,
             const Blob& token,
             const Value::Id& vid);
     void onAnnounceDone(const Sp<Node>& status,
-            net::NetworkEngine::RequestAnswer& a,
+            net::RequestAnswer& a,
             Sp<Search>& sr);
 };
 

include/opendht/infohash.h

@@ -250,17 +250,24 @@ public:
         num ^= (-b ^ num) & (1 << bit);
     }
 
-    double
-    toFloat() const
-    {
+    double toFloat() const {
+        using D = size_t;
         double v = 0.;
-        for (unsigned i = 0; i < std::min<size_t>(HASH_LEN, sizeof(unsigned)-1); i++)
-            v += *(cbegin()+i)/(double)(1<<(8*(i+1)));
+        for (size_t i = 0; i < std::min<size_t>(HASH_LEN, sizeof(D)-1); i++)
+            v += *(cbegin()+i) / (double)((D)1 << 8*(i+1));
         return v;
     }
 
-    bool
-    operator<(const InfoHash& o) const {
+    bool operator==(const InfoHash& h) const {
+        auto a = reinterpret_cast<const uint32_t*>(data());
+        auto b = reinterpret_cast<const uint32_t*>(h.data());
+        for (size_t i=0; i<(size()/sizeof(uint32_t)); i++)
+            if (a[i] != b[i])
+                return false;
+        return true;
+    }
+
+    bool operator<(const InfoHash& o) const {
         for(unsigned i = 0; i < HASH_LEN; i++) {
             if((*this)[i] != o[i])
                 return (*this)[i] < o[i];

include/opendht/log.h

@@ -24,6 +24,10 @@
 #include <iostream>
 
 namespace dht {
+
+/**
+ * Logging-related functions
+ */
 namespace log {
 
 /**

include/opendht/net.h

@@ -42,41 +42,26 @@ struct TransPrefix : public std::array<uint8_t, 2> {
 struct TransId final : public std::array<uint8_t, 4> {
     static const constexpr uint16_t INVALID {0};
 
-    TransId() { std::fill_n(begin(), 4, 0); }
+    TransId() { std::fill(begin(), end(), 0); }
     TransId(const std::array<char, 4>& o) { std::copy(o.begin(), o.end(), begin()); }
     TransId(const TransPrefix prefix, uint16_t seqno = 0) {
         std::copy_n(prefix.begin(), prefix.size(), begin());
         *reinterpret_cast<uint16_t*>(data()+prefix.size()) = seqno;
     }
-
-    TransId(const char* q, size_t l) : array<uint8_t, 4>() {
-        if (l > 4) {
-            length = 0;
-        } else {
-            std::copy_n(q, l, begin());
-            length = l;
-        }
-    }
-
-    uint16_t getTid() const {
-        return *reinterpret_cast<const uint16_t*>(&(*this)[2]);
+    TransId(uint32_t id) {
+        *reinterpret_cast<uint32_t*>(data()) = htonl(id);
     }
 
     uint32_t toInt() const {
-        return *reinterpret_cast<const uint32_t*>(&(*this)[0]);
+        return ntohl(*reinterpret_cast<const uint32_t*>(&(*this)[0]));
     }
 
-    bool matches(const TransPrefix prefix, uint16_t* tid = nullptr) const {
-        if (std::equal(begin(), begin()+2, prefix.begin())) {
-            if (tid)
-                *tid = getTid();
-            return true;
-        } else
-            return false;
+    bool matches(const TransPrefix prefix) const {
+        return std::equal(begin(), begin()+2, prefix.begin());
     }
-
-    unsigned length {4};
 };
 
+TransId unpackTid(msgpack::object& o);
+
 } /* namespace net */
 } /* dht */

include/opendht/network_engine.h

@@ -36,7 +36,6 @@
 #include <functional>
 #include <algorithm>
 #include <memory>
-#include <random>
 #include <queue>
 
 namespace dht {
@@ -83,6 +82,20 @@ private:
 
 struct ParsedMessage;
 
+/**
+ * Answer for a request.
+ */
+struct RequestAnswer {
+    Blob ntoken {};
+    Value::Id vid {};
+    std::vector<Sp<Value>> values {};
+    std::vector<Sp<FieldValueIndex>> fields {};
+    std::vector<Sp<Node>> nodes4 {};
+    std::vector<Sp<Node>> nodes6 {};
+    RequestAnswer() {}
+    RequestAnswer(ParsedMessage&& msg);
+};
+
 /*!
  * @class   NetworkEngine
  * @brief   An abstraction of communication protocol on the network.
@@ -103,26 +116,6 @@ struct ParsedMessage;
  */
 class NetworkEngine final
 {
-public:
-    /*!
-     * @class   RequestAnswer
-     * @brief   Answer for a request.
-     * @details
-     * Answer for a request to be (de)serialized. Used for reponding to a node
-     * and looking up the response from a node.
-     */
-    struct RequestAnswer {
-        Blob ntoken {};
-        Value::Id vid {};
-        std::vector<Sp<Value>> values {};
-        std::vector<Sp<FieldValueIndex>> fields {};
-        std::vector<Sp<Node>> nodes4 {};
-        std::vector<Sp<Node>> nodes6 {};
-        RequestAnswer() {}
-        RequestAnswer(ParsedMessage&& msg);
-    };
-
-
 private:
     /**
      * Called when we receive an error message.
@@ -208,7 +201,6 @@ private:
             const Value::Id&)> onRefresh {};
 
 public:
-    using SocketCb = std::function<void(const Sp<Node>&, RequestAnswer&&)>;
     using RequestCb = std::function<void(const Request&, RequestAnswer&&)>;
     using RequestExpiredCb = std::function<void(const Request&, bool)>;
 
@@ -257,12 +249,6 @@ public:
     bool isRunning(sa_family_t af) const;
     inline want_t want () const { /*return dht_socket >= 0 && dht_socket6 >= 0 ? (WANT4 | WANT6) : -1;*/return WANT4 | WANT6; }
 
-    /**
-     * Cancel a request. Setting req->cancelled = true is not enough in the case
-     * a request is "persistent".
-     */
-    void cancelRequest(Sp<Request>& req);
-
     void connectivityChanged(sa_family_t);
 
     /**************
@@ -403,23 +389,6 @@ public:
                                  const Blob& token,
                                  RequestCb&& on_done,
                                  RequestExpiredCb&& on_expired);
-    /**
-     * Opens a socket on which a node will be able allowed to write for further
-     * additionnal updates following the response to a previous request.
-     *
-     * @param node  The node which will be allowed to write on this socket.
-     * @param cb    The callback to execute once updates arrive on the socket.
-     *
-     * @return the socket.
-     */
-    Sp<Socket> openSocket(const Sp<Node>& node, TransPrefix tp, SocketCb&& cb);
-
-    /**
-     * Closes a socket so that no further data will be red on that socket.
-     *
-     * @param socket  The socket to close.
-     */
-    void closeSocket(Sp<Socket> socket);
 
     Sp<Node> insertNode(const InfoHash& myid, const SockAddr& addr) {
         auto n = cache.getNode(myid, addr, {}, scheduler.time(), 0);
@@ -451,7 +420,7 @@ private:
     /***************
      *  Constants  *
      ***************/
-    static constexpr long unsigned MAX_REQUESTS_PER_SEC {1600};
+    static constexpr size_t MAX_REQUESTS_PER_SEC {1600};
     /* the length of a node info buffer in ipv4 format */
     static const constexpr size_t NODE4_INFO_BUF_LEN {HASH_LEN + sizeof(in_addr) + sizeof(in_port_t)};
     /* the length of a node info buffer in ipv6 format */
@@ -471,7 +440,6 @@ private:
     static constexpr size_t MAX_PACKET_VALUE_SIZE {600};
 
     static const std::string my_v;
-    static std::mt19937 rd_device;
 
     /**
      * Parses a message and calls appropriate callbacks.
@@ -499,17 +467,7 @@ private:
      * Sends a request to a node. Request::MAX_ATTEMPT_COUNT attempts will
      * be made before the request expires.
      */
-    void sendRequest(Sp<Request>& request);
-
-    /**
-     * Generates a new request id, skipping the invalid id.
-     *
-     * @return the new id.
-     */
-    uint16_t getNewTid() {
-        ++transaction_id;
-        return transaction_id == TransId::INVALID ? ++transaction_id : transaction_id;
-    }
+    void sendRequest(const Sp<Request>& request);
 
     struct MessageStats {
         unsigned ping    {0};
@@ -524,7 +482,7 @@ private:
     int send(msgpack::sbuffer& msg, const Node& node, const Sp<TcpSocket>& sock = {});
     int sendUDP(msgpack::sbuffer& msg, const SockAddr& addr);
 
-    int send(const Blob& msg, int flags, const Node& node);
+    int send(const Blob& msg, int flags, const Node& node, UdpSocket::OnSent&& cb);
 
     void startTcp(const Sp<TcpSocket>& sock, const Sp<Node>& node = {});
 
@@ -579,45 +537,16 @@ private:
     crypto::EcSecretKey id_key;
     //crypto::EcSecretKey id_key;
 
-    /**
-     * A comparator to classify IP addresses, only considering the
-     * first 64 bits in IPv6.
-     */
-    struct cmpSockAddr {
-        bool operator()(const SockAddr& a, const SockAddr& b) const {
-            if (a.second != b.second)
-                return a.second < b.second;
-            socklen_t start, len;
-            switch(a.getFamily()) {
-                case AF_INET:
-                    start = offsetof(sockaddr_in, sin_addr);
-                    len = sizeof(in_addr);
-                    break;
-                case AF_INET6:
-                    start = offsetof(sockaddr_in6, sin6_addr);
-                    // don't consider more than 64 bits (IPv6)
-                    len = 8;
-                    break;
-                default:
-                    start = 0;
-                    len = a.second;
-                    break;
-            }
-
-            return std::memcmp((uint8_t*)&a.first+start, (uint8_t*)&b.first+start, len) < 0;
-        }
-    };
     // global limiting should be triggered by at least 8 different IPs
     using IpLimiter = RateLimiter<MAX_REQUESTS_PER_SEC/8>;
     using IpLimiterMap = std::map<SockAddr, IpLimiter, SockAddr::ipCmp>;
     IpLimiterMap address_rate_limiter {};
     RateLimiter<MAX_REQUESTS_PER_SEC> rate_limiter {};
+    size_t limiter_maintenance {0};
 
     // requests handling
-    uint16_t transaction_id {1};
     std::map<TransId, Sp<Request>> requests {};
     std::map<TransId, PartialMessage> partial_messages;
-    std::map<TransId, Sp<Socket>> opened_sockets {};
 
     MessageStats in_stats {}, out_stats {};
     std::set<SockAddr> blacklist {};

include/opendht/node.h

@@ -22,31 +22,42 @@
 #include "infohash.h" // includes socket structures
 #include "utils.h"
 #include "sockaddr.h"
+#include "net.h"
 #include "uv_utils.h"
 #include "crypto.h"
 
 #include <list>
+#include <map>
 
 namespace dht {
 
+struct Node;
 namespace net {
 struct Request;
+struct Socket;
+struct RequestAnswer;
 } /* namespace net */
 
+using SocketCb = std::function<void(const Sp<Node>&, net::RequestAnswer&&)>;
+using SocketId = uint32_t;
+struct Socket {
+    Socket() {}
+    Socket(SocketCb&& on_receive) :
+        on_receive(std::move(on_receive)) {}
+    SocketCb on_receive {};
+};
+
 struct Node {
-    InfoHash id;
-    SockAddr addr;
+    const InfoHash id;
+
     Sp<TcpSocket> sock;
 
     InfoHash last_known_pk;
     //std::list<crypto::EcSecretKey>
 
-    time_point time {time_point::min()};            /* last time eared about */
-    time_point reply_time {time_point::min()};      /* time of last correct reply received */
-
+    Node(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& s = {}, bool client=false);
     Node(const InfoHash& id, const sockaddr* sa, socklen_t salen)
-        : id(id), addr(sa, salen) {}
-    Node(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& s = {}) : id(id), addr(addr), sock(s) {}
+        : Node(id, SockAddr(sa, salen)) {}
     ~Node() {
         if (sock)
             sock->close();
@@ -55,12 +66,18 @@ struct Node {
     InfoHash getId() const {
         return id;
     }
-    std::pair<const sockaddr*, socklen_t> getAddr() const {
-        return {(const sockaddr*)&addr.first, addr.second};
-    }
+    const SockAddr& getAddr() const { return addr; }
     std::string getAddrStr() const {
         return addr.toString();
     }
+
+    bool isClient() const { return is_client; }
+    bool isIncoming() { return time > reply_time; }
+
+    const time_point& getTime() const { return time; }
+    const time_point& getReplyTime() const { return reply_time; }
+    void setTime(const time_point& t) { time = t; }
+
     SockAddr getAddress() const {
         return canStream() ? sock->getPeerAddr() : addr;
     }
@@ -85,7 +102,20 @@ struct Node {
     bool isPendingMessage() const;
     size_t getPendingMessageCount() const;
 
-    NodeExport exportNode() const { return NodeExport {id, addr.first, addr.second}; }
+    bool isOld(const time_point& now) const {
+        return time + NODE_EXPIRE_TIME < now;
+    }
+    bool isRemovable(const time_point& now) const {
+        return isExpired() and isOld(now);
+    }
+
+    NodeExport exportNode() const {
+        NodeExport ne;
+        ne.id = id;
+        ne.sslen = addr.getLength();
+        std::memcpy(&ne.ss, addr.get(), ne.sslen);
+        return ne;
+    }
     sa_family_t getFamily() const { return addr.getFamily(); }
 
     void update(const SockAddr&, const Sp<TcpSocket>&);
@@ -94,16 +124,48 @@ struct Node {
         return sock and not sock->isClosed() and sock->canWrite();
     }
 
-    void requested(Sp<net::Request>& req);
-    void received(time_point now, Sp<net::Request> req);
+    void requested(const Sp<net::Request>& req);
+    void received(time_point now, const Sp<net::Request>& req);
+    Sp<net::Request> getRequest(const net::TransId& tid);
+    void cancelRequest(const Sp<net::Request>& req);
 
     void setExpired();
 
+    /**
+     * Opens a socket on which a node will be able allowed to write for further
+     * additionnal updates following the response to a previous request.
+     *
+     * @param node  The node which will be allowed to write on this socket.
+     * @param cb    The callback to execute once updates arrive on the socket.
+     *
+     * @return the socket.
+     */
+    SocketId openSocket(SocketCb&& cb);
+
+    Socket* getSocket(SocketId id);
+
+    /**
+     * Closes a socket so that no further data will be red on that socket.
+     *
+     * @param socket  The socket to close.
+     */
+    void closeSocket(SocketId id);
+
     /**
      * Resets the state of the node so it's not expired anymore.
      */
     void reset() { expired_ = false; reply_time = time_point::min(); }
 
+    /**
+     * Generates a new request id, skipping the invalid id.
+     *
+     * @return the new id.
+     */
+    uint16_t getNewTid() {
+        ++transaction_id;
+        return transaction_id == net::TransId::INVALID ? ++transaction_id : transaction_id;
+    }
+
     std::string toString() const;
 
     OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const Node& h);
@@ -120,15 +182,17 @@ private:
     /* Number of times we accept authentication errors from this node. */
     static const constexpr unsigned MAX_AUTH_ERRORS {3};
 
-    std::list<std::weak_ptr<net::Request>> requests_ {};
+    SockAddr addr;
+    bool is_client {false};
+    time_point time {time_point::min()};            /* last time eared about */
+    time_point reply_time {time_point::min()};      /* time of last correct reply received */
     unsigned auth_errors {0};
     bool expired_ {false};
+    SocketId transaction_id;
+    using TransactionDist = std::uniform_int_distribution<decltype(transaction_id)>;
 
-    void clearPendingQueue() {
-        requests_.remove_if([](std::weak_ptr<net::Request>& w) {
-            return w.expired();
-        });
-    }
+    std::map<net::TransId, Sp<net::Request>> requests_ {};
+    std::map<SocketId, Socket> sockets_;
 };
 
 }

include/opendht/node_cache.h

@@ -26,9 +26,9 @@
 namespace dht {
 
 struct NodeCache {
-    std::shared_ptr<Node> getNode(const InfoHash& id, sa_family_t family);
-    std::shared_ptr<Node> getNode(const InfoHash& id, const SockAddr&, const Sp<TcpSocket>&, time_point now, bool confirmed);
-    std::vector<std::shared_ptr<Node>> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count);
+    Sp<Node> getNode(const InfoHash& id, sa_family_t family);
+    Sp<Node> getNode(const InfoHash& id, const SockAddr&, const Sp<TcpSocket>&, time_point now, bool confirmed, bool client=false);
+    std::vector<Sp<Node>> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const;
 
     /**
      * Reset the connectivity state of every node,
@@ -41,12 +41,14 @@ struct NodeCache {
 private:
     class NodeMap : public std::map<InfoHash, std::weak_ptr<Node>> {
     public:
-        std::shared_ptr<Node> getNode(const InfoHash& id);
-        std::shared_ptr<Node> getNode(const InfoHash& id, const SockAddr&, const Sp<TcpSocket>&, time_point now, bool confirmed);
+        Sp<Node> getNode(const InfoHash& id);
+        Sp<Node> getNode(const InfoHash& id, const SockAddr&, const Sp<TcpSocket>&, time_point now, bool confirmed, bool client);
         void clearBadNodes();
         void closeAll();
     };
 
+    const NodeMap& cache(sa_family_t af) const { return af == AF_INET ? cache_4 : cache_6; }
+    NodeMap& cache(sa_family_t af) { return af == AF_INET ? cache_4 : cache_6; }
     NodeMap cache_4;
     NodeMap cache_6;
 };

include/opendht/rate_limiter.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016 Savoir-faire Linux Inc.
+ *  Copyright (C) 2017 Savoir-faire Linux Inc.
  *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
  *
  *  This program is free software; you can redistribute it and/or modify
@@ -26,10 +26,10 @@ namespace dht {
 template<size_t Quota, unsigned long Perdiod=1>
 class RateLimiter {
 public:
-    /** Clear outdated records and return current quota usage */ 
+    /** Clear outdated records and return current quota usage */
     size_t maintain(const time_point& now) {
-        using namespace std::chrono;
-        while (not records.empty() and duration_cast<seconds>(now - records.front()) >= seconds(Perdiod))
+        auto limit = now - std::chrono::seconds(Perdiod);
+        while (not records.empty() and records.front() < limit)
             records.pop();
         return records.size();
     }

include/opendht/routing_table.h

@@ -24,6 +24,9 @@
 namespace dht {
 
 static constexpr unsigned TARGET_NODES {8};
+namespace net {
+class NetworkEngine;
+}
 
 struct Bucket {
     Bucket() : cached() {}
@@ -38,6 +41,8 @@ struct Bucket {
     /** Return a random node in a bucket. */
     Sp<Node> randomNode();
 
+    void sendCachedPing(net::NetworkEngine& ne);
+
     /** Return false if there is a single non-expired node in this bucket. */
     bool expired() const {
         for (const auto& n : nodes)
@@ -51,6 +56,9 @@ class RoutingTable : public std::list<Bucket> {
 public:
     using std::list<Bucket>::list;
 
+    time_point grow_time {time_point::min()};
+    bool is_client {false};
+
     InfoHash middle(const RoutingTable::const_iterator&) const;
 
     std::vector<Sp<Node>> findClosestNodes(const InfoHash id, time_point now, size_t count = TARGET_NODES) const;
@@ -73,6 +81,14 @@ public:
         return empty() || (size() == 1 && front().nodes.empty());
     }
 
+    void connectivityChanged(const time_point& now) {
+        grow_time = now;
+        for (auto& b : *this)
+            b.time = time_point::min();
+    }
+
+    bool onNewNode(const Sp<Node>& node, int comfirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne);
+
     /**
      * Return a random id in the bucket's range.
      */

include/opendht/securedht.h

@@ -158,7 +158,7 @@ private:
 const ValueType CERTIFICATE_TYPE = {
     8, "Certificate", std::chrono::hours(24 * 7),
     // A certificate can only be stored at its public key ID.
-    [](InfoHash id, Sp<Value>& v, InfoHash, const sockaddr*, socklen_t) {
+    [](InfoHash id, Sp<Value>& v, const InfoHash&, const SockAddr&) {
         try {
             crypto::Certificate crt(v->data);
             // TODO check certificate signature
@@ -166,7 +166,7 @@ const ValueType CERTIFICATE_TYPE = {
         } catch (const std::exception& e) {}
         return false;
     },
-    [](InfoHash, const Sp<Value>& o, Sp<Value>& n, InfoHash, const sockaddr*, socklen_t) {
+    [](InfoHash, const Sp<Value>& o, Sp<Value>& n, const InfoHash&, const SockAddr&) {
         try {
             return crypto::Certificate(o->data).getPublicKey().getId() == crypto::Certificate(n->data).getPublicKey().getId();
         } catch (const std::exception& e) {}

include/opendht/sockaddr.h

@@ -34,65 +34,116 @@ typedef uint16_t sa_family_t;
 typedef uint16_t in_port_t;
 #endif
 
+#include <string>
+#include <memory>
+
+#include <cstring>
+#include <cstddef>
+
 namespace dht {
 
 OPENDHT_PUBLIC std::string print_addr(const sockaddr* sa, socklen_t slen);
 OPENDHT_PUBLIC std::string print_addr(const sockaddr_storage& ss, socklen_t sslen);
 
-struct OPENDHT_PUBLIC SockAddr : public std::pair<sockaddr_storage, socklen_t> {
+/**
+ * A Socket Address (sockaddr*), with abstraction for IPv4, IPv6 address families.
+ */
+class OPENDHT_PUBLIC SockAddr {
 public:
-    SockAddr() : pair<sockaddr_storage, socklen_t>::pair({},0) {}
-    SockAddr(const SockAddr& o) : pair<sockaddr_storage, socklen_t>::pair({},o.second) {
-        std::copy_n((uint8_t*)&o.first, o.second, (uint8_t*)&first);
+    SockAddr() {}
+    SockAddr(const SockAddr& o) {
+        set(o.get(), o.getLength());
     }
-    SockAddr(const sockaddr* sa, socklen_t len) : pair<sockaddr_storage, socklen_t>::pair({},len) {
-        if (len > sizeof(sockaddr_storage))
+    SockAddr(SockAddr&& o) : len(o.len), addr(std::move(o.addr)) {
+        o.len = 0;
+    }
+
+    /**
+     * Build from existing address.
+     */
+    SockAddr(const sockaddr* sa, socklen_t length) {
+        if (length > sizeof(sockaddr_storage))
             throw std::runtime_error("Socket address length is too large");
-        std::copy_n((uint8_t*)sa, len, (uint8_t*)&first);
+        set(sa, length);
     }
-    SockAddr(const sockaddr* sa) : pair<sockaddr_storage, socklen_t>::pair({},0) {
+    SockAddr(const sockaddr* sa) {
+        socklen_t len = 0;
         if (sa) {
             if (sa->sa_family == AF_INET)
-                second = sizeof(sockaddr_in);
+                len = sizeof(sockaddr_in);
             else if(sa->sa_family == AF_INET6)
-                second = sizeof(sockaddr_in6);
+                len = sizeof(sockaddr_in6);
+            else
+                throw std::runtime_error("Unknown address family");
         }
-        std::copy_n((uint8_t*)sa, second, (uint8_t*)&first);
+        set(sa, len);
     }
+
+    /**
+     * Build from an existing sockaddr_storage structure.
+     */
     SockAddr(const sockaddr_storage& ss, socklen_t len) : SockAddr((const sockaddr*)&ss, len) {}
 
     bool operator<(const SockAddr& o) const {
-        if (second != o.second)
-            return second < o.second;
-        return std::memcmp((uint8_t*)&first, (uint8_t*)&o.first, second) < 0;
+        if (len != o.len)
+            return len < o.len;
+        return std::memcmp((const uint8_t*)get(), (const uint8_t*)o.get(), len) < 0;
     }
 
     bool equals(const SockAddr& o) const {
-        return second == o.second
-            && std::memcmp((uint8_t*)&first, (uint8_t*)&o.first, second) == 0;
+        return len == o.len
+            && std::memcmp((const uint8_t*)get(), (const uint8_t*)o.get(), len) == 0;
     }
     SockAddr& operator=(const SockAddr& o) {
-        std::copy_n((const uint8_t*)&o.first, o.second, (uint8_t*)&first);
-        second = o.second;
+        set(o.get(), o.getLength());
+        return *this;
+    }
+    SockAddr& operator=(SockAddr&& o) {
+        len = o.len;
+        o.len = 0;
+        addr = std::move(o.addr);
         return *this;
     }
 
     std::string toString() const {
-        return print_addr(first, second);
-    }
-    sa_family_t getFamily() const { return second > sizeof(sa_family_t) ? first.ss_family : AF_UNSPEC; }
-    void setFamily(sa_family_t af) {
-        first.ss_family = af;
-        switch(af) {
-        case AF_INET:
-            second = sizeof(sockaddr_in);
-            break;
-        case AF_INET6:
-            second = sizeof(sockaddr_in6);
-            break;
-        }
+        return print_addr(get(), getLength());
     }
 
+    /**
+     * Returns the address family or AF_UNSPEC if the address is not set.
+     */
+    sa_family_t getFamily() const { return len > sizeof(sa_family_t) ? addr->sa_family : AF_UNSPEC; }
+
+    /**
+     * Resize the managed structure to the appropriate size (if needed),
+     * in which case the sockaddr structure is cleared to zero,
+     * and set the address family field (sa_family).
+     */
+    void setFamily(sa_family_t af) {
+        socklen_t new_length;
+        switch(af) {
+        case AF_INET:
+            new_length = sizeof(sockaddr_in);
+            break;
+        case AF_INET6:
+            new_length = sizeof(sockaddr_in6);
+            break;
+        default:
+            new_length = 0;
+        }
+        if (new_length != len) {
+            len = new_length;
+            if (len) addr.reset((sockaddr*)std::calloc(len, 1));
+            else     addr.reset();
+        }
+        if (len > sizeof(sa_family_t))
+            addr->sa_family = af;
+    }
+
+    /**
+     * Retreive the port (in host byte order) or 0 if the address is not
+     * of a supported family.
+     */
     in_port_t getPort() const {
         switch(getFamily()) {
         case AF_INET:
@@ -103,6 +154,10 @@ public:
             return 0;
         }
     }
+    /**
+     * Set the port. The address must be of a supported family.
+     * @param p The port in host byte order.
+     */
     void setPort(in_port_t p) {
         switch(getFamily()) {
         case AF_INET:
@@ -114,23 +169,42 @@ public:
         }
     }
 
+    /**
+     * Returns the accessible byte length at the pointer returned by #get().
+     * If zero, #get() returns null.
+     */
+    socklen_t getLength() const { return len; }
+
+    /**
+     * An address is defined to be true if its length is not zero.
+     */
+    explicit operator bool() const noexcept {
+        return len;
+    }
+
+    /**
+     * Returns the address to the managed sockaddr structure.
+     * The accessible length is returned by #getLength().
+     */
+    const sockaddr* get() const { return addr.get(); }
+
+    /**
+     * Returns the address to the managed sockaddr structure.
+     * The accessible length is returned by #getLength().
+     */
+    sockaddr* get() { return addr.get(); }
+
     const sockaddr_in& getIPv4() const {
-        return *reinterpret_cast<const sockaddr_in*>(&first);
+        return *reinterpret_cast<const sockaddr_in*>(get());
     }
     const sockaddr_in6& getIPv6() const {
-        return *reinterpret_cast<const sockaddr_in6*>(&first);
+        return *reinterpret_cast<const sockaddr_in6*>(get());
     }
     sockaddr_in& getIPv4() {
-        return *reinterpret_cast<sockaddr_in*>(&first);
+        return *reinterpret_cast<sockaddr_in*>(get());
     }
     sockaddr_in6& getIPv6() {
-        return *reinterpret_cast<sockaddr_in6*>(&first);
-    }
-    sockaddr* get() {
-        return reinterpret_cast<sockaddr*>(&first);
-    }
-    const sockaddr* get() const {
-        return reinterpret_cast<const sockaddr*>(&first);
+        return *reinterpret_cast<sockaddr_in6*>(get());
     }
 
     /**
@@ -154,8 +228,8 @@ public:
      */
     struct ipCmp {
         bool operator()(const SockAddr& a, const SockAddr& b) const {
-            if (a.second != b.second)
-                return a.second < b.second;
+            if (a.len != b.len)
+                return a.len < b.len;
             socklen_t start, len;
             switch(a.getFamily()) {
                 case AF_INET:
@@ -169,18 +243,30 @@ public:
                     break;
                 default:
                     start = 0;
-                    len = a.second;
+                    len = a.len;
                     break;
             }
-
-            return std::memcmp((uint8_t*)&a.first+start, (uint8_t*)&b.first+start, len) < 0;
+            return std::memcmp((uint8_t*)a.get()+start,
+                               (uint8_t*)b.get()+start, len) < 0;
         }
     };
+private:
+    socklen_t len {0};
+    struct free_delete { void operator()(void* p) { std::free(p); } };
+    std::unique_ptr<sockaddr, free_delete> addr {};
+
+    void set(const sockaddr* sa, socklen_t length) {
+        if (len != length) {
+            len = length;
+            if (len) addr.reset((sockaddr*)std::malloc(len));
+            else     addr.reset();
+        }
+        if (len)
+            std::memcpy((uint8_t*)get(), (const uint8_t*)sa, len);
+    }
 
 };
 
 OPENDHT_PUBLIC bool operator==(const SockAddr& a, const SockAddr& b);
 
-OPENDHT_PUBLIC std::string printAddr(const SockAddr& addr);
-
 }

include/opendht/utils.h

@@ -32,6 +32,9 @@
 #define WANT4 1
 #define WANT6 2
 
+/**
+ * OpenDHT C++ namespace
+ */
 namespace dht {
 
 using NetId = uint32_t;
@@ -93,6 +96,9 @@ public:
 
 // Serialization related definitions and utility functions
 
+/**
+ * Arbitrary binary data.
+ */
 using Blob = std::vector<uint8_t>;
 
 /**

include/opendht/uv_utils.h

@@ -132,6 +132,8 @@ using OnClose = std::function<void()>;
 struct UdpSocket : public std::enable_shared_from_this<UdpSocket>
 {
     using OnReceive = std::function<void(const uint8_t* data, size_t size, const SockAddr& addr)>;
+    using OnSent = std::function<void(int status)>;
+
     UdpSocket(uv_loop_t* loop) {
         uv_udp_init(loop, &sock);
         sock.data = nullptr;
@@ -139,17 +141,18 @@ struct UdpSocket : public std::enable_shared_from_this<UdpSocket>
     void open(const char* bind_addr, in_port_t port, OnReceive cb) {
         if (not sock.data)
             sock.data = new std::shared_ptr<UdpSocket>(shared_from_this());
-        receive_cb = cb;
+        receive_cb = std::move(cb);
         struct sockaddr_in6 addr;
         uv_ip6_addr(bind_addr, port, &addr);
         uv_udp_bind(&sock, (const struct sockaddr*)&addr, 0);
         uv_udp_recv_start(&sock, &UdpSocket::on_alloc_buffer, &UdpSocket::on_receive);
     }
 
-    int send(uint8_t* data, size_t size, const SockAddr& addr) {
+    int send(uint8_t* data, size_t size, const SockAddr& addr, OnSent&& cb = {}) {
         auto req = new SendReq;
         req->msg = data;
         req->req.data = req;
+        req->cb = std::move(cb);
         const uv_buf_t buf {
             /*.base =*/ (char*)data,
             /*.len =*/ size
@@ -165,16 +168,16 @@ struct UdpSocket : public std::enable_shared_from_this<UdpSocket>
         return uv_is_active((uv_handle_t*) &sock);
     }
     SockAddr getBoundAddr() {
-        SockAddr addr;
-        int size = sizeof(addr.first);
-        uv_udp_getsockname(&sock, addr.get(), &size);
-        addr.second = size;
-        return addr;
+        sockaddr_storage addr;
+        int size = sizeof(addr);
+        uv_udp_getsockname(&sock, (sockaddr*)&addr, &size);
+        return {(const sockaddr*)&addr, (socklen_t)size};
     }
 private:
     struct SendReq {
         uv_udp_send_t req;
         uint8_t* msg;
+        OnSent cb;
     };
 
     uv_udp_t sock;
@@ -200,8 +203,10 @@ private:
             delete s;
         }
     }
-    void onSent(SendReq& req, int /*status*/) {
-        // if (status) fprintf(stderr, "onSent %s\n", uv_strerror(status));
+    void onSent(SendReq& req, int status) {
+        if (req.cb) {
+            req.cb(status);
+        }
         free(req.msg);
     }
     static std::shared_ptr<UdpSocket> get(uv_handle_t* handle) {
@@ -290,18 +295,16 @@ struct TcpSocket : public std::enable_shared_from_this<TcpSocket>
         return uv_is_writable((const uv_stream_t*)&sock);
     }
     SockAddr getBoundAddr() {
-        SockAddr addr;
-        int size = sizeof(addr.first);
-        uv_tcp_getsockname(&sock, addr.get(), &size);
-        addr.second = size;
-        return addr;
+        sockaddr_storage addr;
+        int size = sizeof(addr);
+        uv_tcp_getsockname(&sock, (sockaddr*)&addr, &size);
+        return {(const sockaddr*)&addr, (socklen_t)size};
     }
     SockAddr getPeerAddr() {
-        SockAddr addr;
-        int size = sizeof(addr.first);
-        uv_tcp_getpeername(&sock, addr.get(), &size);
-        addr.second = size;
-        return addr;
+        sockaddr_storage addr;
+        int size = sizeof(addr);
+        uv_tcp_getpeername(&sock, (sockaddr*)&addr, &size);
+        return {(const sockaddr*)&addr, (socklen_t)size};
     }
 private:
     uv_tcp_t sock;
@@ -318,7 +321,7 @@ private:
         uint8_t* msg;
         std::shared_ptr<TcpSocket> sock;
         OnWrite callback;
-    };  
+    };
     std::shared_ptr<TcpSocket> accept() {
         auto s = std::make_shared<TcpSocket>(sock.loop);
         s->sock.data = new std::shared_ptr<TcpSocket>(s);

include/opendht/value.h

@@ -22,6 +22,7 @@
 #include "infohash.h"
 #include "crypto.h"
 #include "utils.h"
+#include "sockaddr.h"
 
 #include <msgpack.hpp>
 
@@ -51,7 +52,7 @@ struct Query;
  * @param form_addr: network address of the incoming request.
  * @param from_len: network address lendth of the incoming request.
  */
-using StorePolicy = std::function<bool(InfoHash key, std::shared_ptr<Value>& value, InfoHash from, const sockaddr* from_addr, socklen_t from_len)>;
+using StorePolicy = std::function<bool(InfoHash key, std::shared_ptr<Value>& value, const InfoHash& from, const SockAddr& addr)>;
 
 /**
  * An edition policy is applied once to every incoming value storage requests,
@@ -67,15 +68,15 @@ using StorePolicy = std::function<bool(InfoHash key, std::shared_ptr<Value>& val
  * @param form_addr: network address of the incoming request.
  * @param from_len: network address lendth of the incoming request.
  */
-using EditPolicy = std::function<bool(InfoHash key, const std::shared_ptr<Value>& old_val, std::shared_ptr<Value>& new_val, InfoHash from, const sockaddr* from_addr, socklen_t from_len)>;
+using EditPolicy = std::function<bool(InfoHash key, const std::shared_ptr<Value>& old_val, std::shared_ptr<Value>& new_val, const InfoHash& from, const SockAddr& addr)>;
 
 static constexpr const size_t MAX_VALUE_SIZE {1024 * 64};
 
 struct OPENDHT_PUBLIC ValueType {
     typedef uint16_t Id;
 
-    static bool DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr<Value>& v, InfoHash, const sockaddr*, socklen_t);
-    static bool DEFAULT_EDIT_POLICY(InfoHash, const std::shared_ptr<Value>&, std::shared_ptr<Value>&, InfoHash, const sockaddr*, socklen_t) {
+    static bool DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr<Value>& v, const InfoHash&, const SockAddr&);
+    static bool DEFAULT_EDIT_POLICY(InfoHash, const std::shared_ptr<Value>&, std::shared_ptr<Value>&, const InfoHash&, const SockAddr&) {
         return false;
     }
 

src/Makefile.am

@@ -9,6 +9,7 @@ libopendht_la_SOURCES  = \
         listener.h \
         request.h \
         search.h \
+        parsed_message.h \
         node_cache.cpp \
         callbacks.cpp \
         routing_table.cpp \
@@ -82,4 +83,3 @@ noinst_HEADERS = \
         @top_builddir@/argon2/src/encoding.h \
         @top_builddir@/argon2/src/thread.h
 endif
-

src/default_types.cpp

@@ -27,14 +27,14 @@ std::ostream& operator<< (std::ostream& s, const DhtMessage& v)
 }
 
 bool
-DhtMessage::storePolicy(InfoHash h, std::shared_ptr<Value>& v, InfoHash f, const sockaddr* fa, socklen_t fas)
+DhtMessage::storePolicy(InfoHash h, std::shared_ptr<Value>& v, const InfoHash& f, const SockAddr& sa)
 {
     try {
         auto msg = unpackMsg<DhtMessage>(v->data);
         if (msg.service.empty())
             return false;
     } catch (const std::exception& e) {}
-    return ValueType::DEFAULT_STORE_POLICY(h, v, f, fa, fas);
+    return ValueType::DEFAULT_STORE_POLICY(h, v, f, sa);
 }
 
 Value::Filter
@@ -54,12 +54,11 @@ DhtMessage::ServiceFilter(std::string s)
 
 std::ostream& operator<< (std::ostream& s, const IpServiceAnnouncement& v)
 {
-    s << "Peer: ";
-    s << "port " << v.getPort();
-
-    if (v.ss.ss_family == AF_INET || v.ss.ss_family == AF_INET6) {
+    if (v.addr) {
+        s << "Peer: ";
+        s << "port " << v.getPort();
         char hbuf[NI_MAXHOST];
-        if (getnameinfo((sockaddr*)&v.ss, sizeof(v.ss), hbuf, sizeof(hbuf), nullptr, 0, NI_NUMERICHOST) == 0) {
+        if (getnameinfo(v.addr.get(), v.addr.getLength(), hbuf, sizeof(hbuf), nullptr, 0, NI_NUMERICHOST) == 0) {
             s << " addr " << std::string(hbuf, strlen(hbuf));
         }
     }
@@ -67,17 +66,17 @@ std::ostream& operator<< (std::ostream& s, const IpServiceAnnouncement& v)
 }
 
 bool
-IpServiceAnnouncement::storePolicy(InfoHash h, std::shared_ptr<Value>& v, InfoHash f, const sockaddr* from, socklen_t fromlen)
+IpServiceAnnouncement::storePolicy(InfoHash h, std::shared_ptr<Value>& v, const InfoHash& f, const SockAddr& sa)
 {
     try {
         auto msg = unpackMsg<IpServiceAnnouncement>(v->data);
         if (msg.getPort() == 0)
             return false;
-        IpServiceAnnouncement sa_addr {from, fromlen};
+        IpServiceAnnouncement sa_addr {sa};
         sa_addr.setPort(msg.getPort());
         // argument v is modified (not the value).
         v = std::make_shared<Value>(IpServiceAnnouncement::TYPE, sa_addr, v->id);
-        return ValueType::DEFAULT_STORE_POLICY(h, v, f, from, fromlen);
+        return ValueType::DEFAULT_STORE_POLICY(h, v, f, sa);
     } catch (const std::exception& e) {}
     return false;
 }

src/dht.cpp

@@ -51,9 +51,9 @@ NodeStatus
 Dht::getStatus(sa_family_t af) const
 {
     const auto& stats = getNodesStats(af);
-    auto& ping = af == AF_INET ? pending_pings4 : pending_pings6;
     if (stats.good_nodes)
         return NodeStatus::Connected;
+    auto& ping = af == AF_INET ? pending_pings4 : pending_pings6;
     if (ping or stats.getKnownNodes())
         return NodeStatus::Connecting;
     return NodeStatus::Disconnected;
@@ -100,11 +100,9 @@ Dht::isRunning(sa_family_t af) const { return network_engine.isRunning(af); }
 const Sp<Node>
 Dht::findNode(const InfoHash& id, sa_family_t af) const
 {
-    const Bucket* b = findBucket(id, af);
-    if (!b)
-        return {};
-    for (const auto& n : b->nodes)
-        if (n->id == id) return n;
+    if (const Bucket* b = findBucket(id, af))
+        for (const auto& n : b->nodes)
+            if (n->id == id) return n;
     return {};
 }
 
@@ -112,11 +110,9 @@ Dht::findNode(const InfoHash& id, sa_family_t af) const
 void
 Dht::sendCachedPing(Bucket& b)
 {
-    if (not b.cached)
-        return;
-    DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", b.cached->toString().c_str());
-    network_engine.sendPing(b.cached, nullptr, nullptr);
-    b.cached = {};
+    if (b.cached)
+        DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", b.cached->toString().c_str());
+    b.sendCachedPing(network_engine);
 }
 
 std::vector<SockAddr>
@@ -127,7 +123,7 @@ Dht::getPublicAddress(sa_family_t family)
     });
     std::vector<SockAddr> ret;
     for (const auto& addr : reported_addr)
-        if (!family || family == addr.second.first.ss_family)
+        if (!family || family == addr.second.getFamily())
             ret.emplace_back(addr.second);
     return ret;
 }
@@ -185,75 +181,8 @@ Dht::reportedAddr(const SockAddr& addr)
 void
 Dht::onNewNode(const Sp<Node>& node, int confirm)
 {
-    auto& list = buckets(node->getFamily());
-    auto b = list.findBucket(node->id);
-    if (b == list.end())
-        return;
-
-    for (auto& n : b->nodes) {
-        if (n == node) {
-            if (confirm)
-                trySearchInsert(node);
-            return;
-        }
-    }
-
-    /* New node. */
-    /* Try adding the node to searches */
-    trySearchInsert(node);
-
-    const auto& now = scheduler.time();
-    bool mybucket = list.contains(b, getNodeId());
-    if (mybucket) {
-        if (node->getFamily() == AF_INET)
-            mybucket_grow_time = now;
-        else
-            mybucket6_grow_time = now;
-        if (b->expired())
-            scheduler.edit(nextNodesConfirmation);
-    }
-
-    if (b->nodes.size() >= TARGET_NODES) {
-        /* Try to get rid of an expired node. */
-        for (auto& n : b->nodes)
-            if (n->isExpired()) {
-                n = node;
-                return;
-            }
-        /* Bucket full.  Ping a dubious node */
-        bool dubious = false;
-        for (auto& n : b->nodes) {
-            /* Pick the first dubious node that we haven't pinged in the
-               last 9 seconds.  This gives nodes the time to reply, but
-               tends to concentrate on the same nodes, so that we get rid
-               of bad nodes fast. */
-            if (not n->isGood(now)) {
-                dubious = true;
-                if (not n->isPendingMessage()) {
-                    DHT_LOG.d(n->id, "[node %s] sending ping to dubious node", n->toString().c_str());
-                    network_engine.sendPing(n, nullptr, nullptr);
-                    break;
-                }
-            }
-        }
-
-        if ((mybucket || (is_bootstrap and list.depth(b) < 6)) && (!dubious || list.size() == 1)) {
-            DHT_LOG.d("Splitting from depth %u", list.depth(b));
-            sendCachedPing(*b);
-            list.split(b);
-            onNewNode(node, 0);
-            return;
-        }
-
-        /* No space for this node.  Cache it away for later. */
-        if (confirm or not b->cached)
-            b->cached = node;
-    } else {
-        auto empty = b->nodes.empty();
-        /* Create a new node. */
-        b->nodes.emplace_front(node);
-        if (empty)
-            updateStatus();
+    if (buckets(node->getFamily()).onNewNode(node, confirm, scheduler.time(), myid, network_engine) or confirm) {
+        trySearchInsert(node);
     }
 }
 
@@ -296,7 +225,7 @@ Dht::expireSearches()
 
 void
 Dht::searchNodeGetDone(const net::Request& req,
-        net::NetworkEngine::RequestAnswer&& answer,
+        net::RequestAnswer&& answer,
         std::weak_ptr<Search> ws,
         Sp<Query> query)
 {
@@ -340,7 +269,7 @@ void Dht::paginate(std::weak_ptr<Search> ws, Sp<Query> query, SearchNode* n) {
     if (not sr) return;
     auto select_q = std::make_shared<Query>(Select {}.field(Value::Field::Id), query ? query->where : Where {});
     auto onSelectDone = [this,ws,query](const net::Request& status,
-                                        net::NetworkEngine::RequestAnswer&& answer) mutable {
+                                        net::RequestAnswer&& answer) mutable {
         // retreive search
         auto sr = ws.lock();
         if (not sr) return;
@@ -471,7 +400,7 @@ void Dht::searchSendAnnounceValue(const Sp<Search>& sr) {
         if (not something_to_announce)
             continue;
 
-        auto onDone = [this,ws](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer)
+        auto onDone = [this,ws](const net::Request& req, net::RequestAnswer&& answer)
         { /* when put done */
             if (auto sr = ws.lock()) {
                 onAnnounceDone(req.node, answer, sr);
@@ -485,7 +414,7 @@ void Dht::searchSendAnnounceValue(const Sp<Search>& sr) {
                     scheduler.edit(sr->nextSearchStep, scheduler.time());
         };
         auto onSelectDone =
-            [this,ws,onDone,onExpired](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer) mutable
+            [this,ws,onDone,onExpired](const net::Request& req, net::RequestAnswer&& answer) mutable
             { /* on probing done */
                 const auto& now = scheduler.time();
                 if (auto sr = ws.lock()) {
@@ -618,40 +547,40 @@ Dht::searchStep(Sp<Search> sr)
                     continue;
                 for (const auto& l : sr->listeners) {
                     const auto& query = l.second.query;
-                    if (n.getListenTime(query) <= now) {
-                        DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'",
-                                sr->id.toString().c_str(), n.node->toString().c_str());
+                    if (n.getListenTime(query) > now)
+                        continue;
+                    DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'",
+                            sr->id.toString().c_str(), n.node->toString().c_str());
 
-                        const auto& r = n.listenStatus.find(query);
-                        auto prev_req = r != n.listenStatus.end() ? r->second : nullptr;
+                    const auto& r = n.listenStatus.find(query);
+                    auto prev_req = r != n.listenStatus.end() ? r->second : nullptr;
 
-                        std::weak_ptr<Search> ws = sr;
-                        n.listenStatus[query] = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req,
-                            [this,ws,query](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer) mutable
-                            { /* on done */
-                                if (auto sr = ws.lock()) {
-                                    onListenDone(req.node, answer, sr);
-                                    scheduler.edit(sr->nextSearchStep, scheduler.time());
-                                }
-                            },
-                            [this,ws,query](const net::Request& req, bool over) mutable
-                            { /* on expired */
-                                if (auto sr = ws.lock()) {
-                                    scheduler.edit(sr->nextSearchStep, scheduler.time());
-                                    if (over)
-                                        if (auto sn = sr->getNode(req.node))
-                                            sn->listenStatus.erase(query);
-                                }
-                            },
-                            [this,ws,query](const Sp<Node>& node, net::NetworkEngine::RequestAnswer&& answer) mutable
-                            { /* on new values */
-                                if (auto sr = ws.lock()) {
-                                    onGetValuesDone(node, answer, sr, query);
-                                    scheduler.edit(sr->nextSearchStep, scheduler.time());
-                                }
+                    std::weak_ptr<Search> ws = sr;
+                    n.listenStatus[query] = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req,
+                        [this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable
+                        { /* on done */
+                            if (auto sr = ws.lock()) {
+                                onListenDone(req.node, answer, sr);
+                                scheduler.edit(sr->nextSearchStep, scheduler.time());
                             }
-                        );
-                    }
+                        },
+                        [this,ws,query](const net::Request& req, bool over) mutable
+                        { /* on expired */
+                            if (auto sr = ws.lock()) {
+                                scheduler.edit(sr->nextSearchStep, scheduler.time());
+                                if (over)
+                                    if (auto sn = sr->getNode(req.node))
+                                        sn->listenStatus.erase(query);
+                            }
+                        },
+                        [this,ws,query](const Sp<Node>& node, net::RequestAnswer&& answer) mutable
+                        { /* on new values */
+                            if (auto sr = ws.lock()) {
+                                onGetValuesDone(node, answer, sr, query);
+                                scheduler.edit(sr->nextSearchStep, scheduler.time());
+                            }
+                        }
+                    );
                 }
                 if (not n.candidate and ++i == LISTEN_NODES)
                     break;
@@ -944,39 +873,34 @@ Dht::cancelListen(const InfoHash& id, size_t token)
     if (st != store.end() && tokenlocal)
         st->second.local_listeners.erase(tokenlocal);
 
-    auto searches_cancel_listen = [&](std::map<InfoHash, Sp<Search>> srs) {
-        for (auto& sp : srs) {
-            auto& s = sp.second;
-            if (s->id != id) continue;
-            auto af_token = s->af == AF_INET ? std::get<1>(it->second) : std::get<2>(it->second);
-            if (af_token == 0)
-                continue;
-            Sp<Query> query;
-            const auto& ll = s->listeners.find(af_token);
-            if (ll != s->listeners.cend())
-                query = ll->second.query;
-            for (auto& sn : s->nodes) {
-                if (s->listeners.empty()) { /* also erase requests for all searchnodes. */
-                    for (auto& ls : sn.listenStatus)
-                        network_engine.cancelRequest(ls.second);
-                    sn.listenStatus.clear();
-                } else if (query) {
-                    auto it = sn.listenStatus.find(query);
-                    if (it != sn.listenStatus.end()) {
-                        network_engine.cancelRequest(it->second);
-                        sn.listenStatus.erase(it);
-                    }
-                }
-            }
-            s->listeners.erase(af_token);
-        }
+    auto searches_cancel_listen = [&id](std::map<InfoHash, Sp<Search>>& srs, size_t token) {
+        auto srp = srs.find(id);
+        if (srp != srs.end() and token)
+            srp->second->cancelListen(token);
     };
-    searches_cancel_listen(searches4);
-    searches_cancel_listen(searches6);
+    searches_cancel_listen(searches4, std::get<1>(it->second));
+    searches_cancel_listen(searches6, std::get<2>(it->second));
     listeners.erase(it);
     return true;
 }
 
+struct OpStatus {
+    struct Status {
+        bool done {false};
+        bool ok {false};
+        Status(bool done=false, bool ok=false) : done(done), ok(ok) {}
+    };
+    Status status;
+    Status status4;
+    Status status6;
+};
+
+template <typename T>
+struct GetStatus : public OpStatus {
+    std::vector<Sp<T>> values;
+    std::vector<Sp<Node>> nodes;
+};
+
 void
 Dht::put(const InfoHash& id, Sp<Value> val, DoneCallback callback, time_point created, bool permanent)
 {
@@ -990,54 +914,38 @@ Dht::put(const InfoHash& id, Sp<Value> val, DoneCallback callback, time_point cr
 
     DHT_LOG.d(id, "put: adding %s -> %s", id.toString().c_str(), val->toString().c_str());
 
-    auto ok = std::make_shared<bool>(false);
-    auto done = std::make_shared<bool>(false);
-    auto done4 = std::make_shared<bool>(false);
-    auto done6 = std::make_shared<bool>(false);
-    auto donecb = [=](std::vector<Sp<Node>>&& nodes) {
+    auto op = std::make_shared<OpStatus>();
+    auto donecb = [callback](std::vector<Sp<Node>>&& nodes, OpStatus& op) {
         // Callback as soon as the value is announced on one of the available networks
-        if (callback && !*done && (*done4 && *done6)) {
-            callback(*ok, std::move(nodes));
-            *done = true;
+        if (callback and not op.status.done and (op.status4.done && op.status6.done)) {
+            callback(op.status4.ok or op.status6.ok, std::move(nodes));
+            op.status.done = true;
         }
     };
     announce(id, AF_INET, val, [=](bool ok4, std::vector<Sp<Node>>&& nodes) {
         DHT_LOG.d(id, "Announce done IPv4 %d", ok4);
-        *done4 = true;
-        *ok |= ok4;
-        donecb(std::move(nodes));
+        auto& o = *op;
+        o.status4 = {true, ok4};
+        donecb(std::move(nodes), o);
     }, created, permanent);
     announce(id, AF_INET6, val, [=](bool ok6, std::vector<Sp<Node>>&& nodes) {
         DHT_LOG.d(id, "Announce done IPv6 %d", ok6);
-        *done6 = true;
-        *ok |= ok6;
-        donecb(std::move(nodes));
+        auto& o = *op;
+        o.status6 = {true, ok6};
+        donecb(std::move(nodes), o);
     }, created, permanent);
 }
 
 template <typename T>
-struct OpStatus {
-    struct Status {
-        bool done {false};
-        bool ok {false};
-    };
-    Status status;
-    Status status4;
-    Status status6;
-    std::vector<Sp<T>> values;
-    std::vector<Sp<Node>> nodes;
-};
-
-template <typename T>
-void doneCallbackWrapper(DoneCallback dcb, std::vector<Sp<Node>>&& nodes, Sp<OpStatus<T>> op) {
-    if (op->status.done)
+void doneCallbackWrapper(DoneCallback dcb, const std::vector<Sp<Node>>& nodes, GetStatus<T>& op) {
+    if (op.status.done)
         return;
-    op->nodes.insert(op->nodes.end(), nodes.begin(), nodes.end());
-    if (op->status.ok || (op->status4.done and op->status6.done)) {
-        bool ok = op->status.ok || op->status4.ok || op->status6.ok;
-        op->status.done = true;
+    op.nodes.insert(op.nodes.end(), nodes.begin(), nodes.end());
+    if (op.status.ok or (op.status4.done and op.status6.done)) {
+        bool ok = op.status.ok or op.status4.ok or op.status6.ok;
+        op.status.done = true;
         if (dcb)
-            dcb(ok, std::move(op->nodes));
+            dcb(ok, std::move(op.nodes));
     }
 }
 
@@ -1046,17 +954,18 @@ bool callbackWrapper(Cb get_cb,
         DoneCallback done_cb,
         const std::vector<Sp<T>>& values,
         std::function<std::vector<Sp<T>>(const std::vector<Sp<T>>&)> add_values,
-        Sp<OpStatus<T>> op)
+        Sp<GetStatus<T>> o)
 {
-    if (op->status.done)
+    auto& op = *o;
+    if (op.status.done)
         return false;
     auto newvals = add_values(values);
     if (not newvals.empty()) {
-        op->status.ok = !get_cb(newvals);
-        op->values.insert(op->values.end(), newvals.begin(), newvals.end());
+        op.status.ok = !get_cb(newvals);
+        op.values.insert(op.values.end(), newvals.begin(), newvals.end());
     }
     doneCallbackWrapper(done_cb, {}, op);
-    return !op->status.ok;
+    return !op.status.ok;
 }
 
 
@@ -1079,7 +988,7 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt
     scheduler.syncTime();
 
     Query q {{}, where};
-    auto op = std::make_shared<OpStatus<Value>>();
+    auto op = std::make_shared<GetStatus<Value>>();
 
     auto f = filter.chain(q.where.getFilter());
     auto add_values = [op,f](const std::vector<Sp<Value>>& values) {
@@ -1102,22 +1011,20 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt
 
     Dht::search(id, AF_INET, gcb, {}, [=](bool ok, std::vector<Sp<Node>>&& nodes) {
         //DHT_LOG_WARN("DHT done IPv4");
-        op->status4.done = true;
-        op->status4.ok = ok;
-        doneCallbackWrapper(donecb, std::move(nodes), op);
+        op->status4 = {true, ok};
+        doneCallbackWrapper(donecb, std::move(nodes), *op);
     }, f, q);
     Dht::search(id, AF_INET6, gcb, {}, [=](bool ok, std::vector<Sp<Node>>&& nodes) {
         //DHT_LOG_WARN("DHT done IPv6");
-        op->status6.done = true;
-        op->status6.ok = ok;
-        doneCallbackWrapper(donecb, std::move(nodes), op);
+        op->status6 = {true, ok};
+        doneCallbackWrapper(donecb, std::move(nodes), *op);
     }, f, q);
 }
 
 void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Query&& q)
 {
     scheduler.syncTime();
-    auto op = std::make_shared<OpStatus<FieldValueIndex>>();
+    auto op = std::make_shared<GetStatus<FieldValueIndex>>();
 
     auto f = q.where.getFilter();
     auto values = getLocal(id, f);
@@ -1151,15 +1058,13 @@ void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Quer
 
     Dht::search(id, AF_INET, {}, qcb, [=](bool ok, std::vector<Sp<Node>>&& nodes) {
         //DHT_LOG_WARN("DHT done IPv4");
-        op->status4.done = true;
-        op->status4.ok = ok;
-        doneCallbackWrapper(done_cb, std::move(nodes), op);
+        op->status4 = {true, ok};
+        doneCallbackWrapper(done_cb, std::move(nodes), *op);
     }, f, q);
     Dht::search(id, AF_INET6, {}, qcb, [=](bool ok, std::vector<Sp<Node>>&& nodes) {
         //DHT_LOG_WARN("DHT done IPv6");
-        op->status6.done = true;
-        op->status6.ok = ok;
-        doneCallbackWrapper(done_cb, std::move(nodes), op);
+        op->status6 = {true, ok};
+        doneCallbackWrapper(done_cb, std::move(nodes), *op);
     }, f, q);
 }
 
@@ -1212,10 +1117,10 @@ Dht::getPut(const InfoHash& id, const Value::Id& vid)
         }
         return Sp<Value> {};
     };
-    auto v4 = find_value(searches4);
-    if (v4) return v4;
-    auto v6 = find_value(searches6);
-    if (v6) return v6;
+    if (auto v4 = find_value(searches4))
+        return v4;
+    if (auto v6 = find_value(searches6))
+        return v6;
     return {};
 }
 
@@ -1280,7 +1185,7 @@ Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v)
                     node_listeners.first->toString().c_str());
             std::vector<Sp<Value>> vals {};
             vals.push_back(v.data);
-            Blob ntoken = makeToken(node_listeners.first->addr, false);
+            Blob ntoken = makeToken(node_listeners.first->getAddr(), false);
             network_engine.tellListener(node_listeners.first, l.second.sid, id, 0, ntoken, {}, {},
                     std::move(vals), l.second.query);
         }
@@ -1288,7 +1193,7 @@ Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v)
 }
 
 bool
-Dht::storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr* sa)
+Dht::storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr& sa)
 {
     const auto& now = scheduler.time();
     created = std::min(created, now);
@@ -1308,7 +1213,7 @@ Dht::storageStore(const InfoHash& id, const Sp<Value>& value, time_point created
 
     StorageBucket* store_bucket {nullptr};
     if (sa)
-        store_bucket = &store_quota.emplace(*sa, StorageBucket{}).first->second;
+        store_bucket = &store_quota.emplace(sa, StorageBucket{}).first->second;
 
     auto store = st->second.store(id, value, created, expiration, store_bucket);
     if (auto vs = store.first) {
@@ -1339,7 +1244,7 @@ Dht::storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t socket_
     if (l == node_listeners->second.end()) {
         auto vals = st->second.get(query.where.getFilter());
         if (not vals.empty()) {
-            network_engine.tellListener(node, socket_id, id, WANT4 | WANT6, makeToken(node->addr, false),
+            network_engine.tellListener(node, socket_id, id, WANT4 | WANT6, makeToken(node->getAddr(), false),
                     buckets4.findClosestNodes(id, now, TARGET_NODES), buckets6.findClosestNodes(id, now, TARGET_NODES),
                     std::move(vals), query);
         }
@@ -1419,15 +1324,12 @@ Dht::connectivityChanged(sa_family_t af)
     updateStatus();
     const auto& now = scheduler.time();
     scheduler.edit(nextNodesConfirmation, now);
-    auto& bucket_grow_time = (af == AF_INET) ? mybucket_grow_time : mybucket6_grow_time;
-    bucket_grow_time = now;
-    for (auto& b : buckets(af))
-        b.time = time_point::min();
+    buckets(af).connectivityChanged(now);
     network_engine.connectivityChanged(af);
     for (auto& sp : searches(af))
         for (auto& sn : sp.second->nodes) {
             for (auto& ls : sn.listenStatus)
-                network_engine.cancelRequest(ls.second);
+                sn.node->cancelRequest(ls.second);
             sn.listenStatus.clear();
         }
     reported_addr.erase(std::remove_if(reported_addr.begin(), reported_addr.end(), [&](const ReportedAddr& addr){
@@ -1481,7 +1383,7 @@ Dht::makeToken(const SockAddr& addr, bool old) const
 bool
 Dht::tokenMatch(const Blob& token, const SockAddr& addr) const
 {
-    if (!addr.second || token.size() != TOKEN_SIZE)
+    if (not addr or token.size() != TOKEN_SIZE)
         return false;
     if (token == makeToken(addr, false))
         return true;
@@ -1500,7 +1402,7 @@ Dht::getNodesStats(sa_family_t af) const
         for (auto& n : b.nodes) {
             if (n->isGood(now)) {
                 stats.good_nodes++;
-                if (n->time > n->reply_time)
+                if (n->isIncoming())
                     stats.incoming_nodes++;
             } else if (not n->isExpired())
                 stats.dubious_nodes++;
@@ -1523,10 +1425,12 @@ Dht::dumpBucket(const Bucket& b, std::ostream& out) const
     out  << std::endl;
     for (auto& n : b.nodes) {
         out << "    Node " << n->toString();
-        if (n->time != n->reply_time)
-            out << " age " << duration_cast<seconds>(now - n->time).count() << ", reply: " << duration_cast<seconds>(now - n->reply_time).count();
+        const auto& t = n->getTime();
+        const auto& r = n->getReplyTime();
+        if (t != r)
+            out << " age " << duration_cast<seconds>(now - t).count() << ", reply: " << duration_cast<seconds>(now - r).count();
         else
-            out << " age " << duration_cast<seconds>(now - n->time).count();
+            out << " age " << duration_cast<seconds>(now - t).count();
         if (n->isExpired())
             out << " [expired]";
         else if (n->isGood(now))
@@ -1621,7 +1525,7 @@ Dht::dumpSearch(const Search& sr, std::ostream& out) const
                 out << "] ";
             }
         }
-        out << n.node->addr.toString() << std::endl;
+        out << n.node->getAddrStr() << std::endl;
     }
 }
 
@@ -1788,6 +1692,8 @@ Dht::Dht(uv_loop_t* loop, Config config)
             std::bind(&Dht::onRefresh, this, _1, _2, _3, _4))
 {
     scheduler.syncTime();
+    buckets4.is_client = config.is_bootstrap;
+    buckets6.is_client = config.is_bootstrap;
     search_id = std::uniform_int_distribution<decltype(search_id)>{}(rd);
 
     uniform_duration_distribution<> time_dis {std::chrono::seconds(3), std::chrono::seconds(5)};
@@ -1969,6 +1875,20 @@ Dht::maintainStorage(decltype(store)::value_type& storage, bool force, DoneCallb
     return announce_per_af;
 }
 
+/*time_point
+Dht::periodic(const uint8_t *buf, size_t buflen, const SockAddr& from)
+{
+    scheduler.syncTime();
+    if (buflen) {
+        try {
+            network_engine.processMessage(buf, buflen, from);
+        } catch (const std::exception& e) {
+            DHT_LOG.e("Can't parse message from %s: %s", from.toString().c_str(), e.what());
+        }
+    }
+    return scheduler.run();
+}*/
+
 void
 Dht::updateStatus()
 {
@@ -2022,9 +1942,9 @@ Dht::confirmNodes()
     soon |= bucketMaintenance(buckets6);
 
     if (!soon) {
-        if (mybucket_grow_time >= now - seconds(150))
+        if (buckets4.grow_time >= now - seconds(150))
             soon |= neighbourhoodMaintenance(buckets4);
-        if (mybucket6_grow_time >= now - seconds(150))
+        if (buckets6.grow_time >= now - seconds(150))
             soon |= neighbourhoodMaintenance(buckets6);
     }
 
@@ -2148,7 +2068,7 @@ Dht::pingNode(const SockAddr& sa, DoneCallback&& cb)
     DHT_LOG.d("Sending ping to %s", sa.toString().c_str());
     auto& count = sa.getFamily() == AF_INET ? pending_pings4 : pending_pings6;
     count++;
-    network_engine.sendPing(sa, [&count,cb](const net::Request& req, net::NetworkEngine::RequestAnswer&&) {
+    network_engine.sendPing(sa, [&count,cb](const net::Request& req, net::RequestAnswer&&) {
         count--;
         if (cb)
             cb(true, {req.node});
@@ -2163,14 +2083,15 @@ Dht::pingNode(const SockAddr& sa, DoneCallback&& cb)
 
 void
 Dht::onError(Sp<net::Request> req, net::DhtProtocolException e) {
+    const auto& node = req->node;
     if (e.getCode() == net::DhtProtocolException::UNAUTHORIZED) {
-        DHT_LOG.e(req->node->id, "[node %s] token flush", req->node->toString().c_str());
-        req->node->authError();
-        network_engine.cancelRequest(req);
-        for (auto& srp : searches(req->node->getFamily())) {
+        DHT_LOG.e(node->id, "[node %s] token flush", node->toString().c_str());
+        node->authError();
+        node->cancelRequest(req);
+        for (auto& srp : searches(node->getFamily())) {
             auto& sr = srp.second;
             for (auto& n : sr->nodes) {
-                if (n.node != req->node) continue;
+                if (n.node != node) continue;
                 n.token.clear();
                 n.last_get_reply = time_point::min();
                 searchSendGetValues(sr);
@@ -2178,32 +2099,30 @@ Dht::onError(Sp<net::Request> req, net::DhtProtocolException e) {
             }
         }
     } else if (e.getCode() == net::DhtProtocolException::NOT_FOUND) {
-        DHT_LOG.e(req->node->id, "[node %s] returned error 404: storage not found", req->node->toString().c_str());
-        network_engine.cancelRequest(req);
+        DHT_LOG.e(node->id, "[node %s] returned error 404: storage not found", node->toString().c_str());
+        node->cancelRequest(req);
     }
 }
 
 void
-Dht::onReportedAddr(const InfoHash& id, const SockAddr& addr)
+Dht::onReportedAddr(const InfoHash& /*id*/, const SockAddr& addr)
 {
-    const auto& b = buckets(addr.getFamily()).findBucket(id);
-    b->time = scheduler.time();
-    if (addr.second)
+    if (addr)
         reportedAddr(addr);
 }
 
-net::NetworkEngine::RequestAnswer
+net::RequestAnswer
 Dht::onPing(Sp<Node>)
 {
     return {};
 }
 
-net::NetworkEngine::RequestAnswer
+net::RequestAnswer
 Dht::onFindNode(Sp<Node> node, const InfoHash& target, want_t want)
 {
     const auto& now = scheduler.time();
-    net::NetworkEngine::RequestAnswer answer;
-    answer.ntoken = makeToken(node->addr, false);
+    net::RequestAnswer answer;
+    answer.ntoken = makeToken(node->getAddr(), false);
     if (want & WANT4)
         answer.nodes4 = buckets4.findClosestNodes(target, now, TARGET_NODES);
     if (want & WANT6)
@@ -2211,7 +2130,7 @@ Dht::onFindNode(Sp<Node> node, const InfoHash& target, want_t want)
     return answer;
 }
 
-net::NetworkEngine::RequestAnswer
+net::RequestAnswer
 Dht::onGetValues(Sp<Node> node, const InfoHash& hash, want_t, const Query& query)
 {
     if (hash == zeroes) {
@@ -2222,9 +2141,9 @@ Dht::onGetValues(Sp<Node> node, const InfoHash& hash, want_t, const Query& query
         };
     }
     const auto& now = scheduler.time();
-    net::NetworkEngine::RequestAnswer answer {};
+    net::RequestAnswer answer {};
     auto st = store.find(hash);
-    answer.ntoken = makeToken(node->addr, false);
+    answer.ntoken = makeToken(node->getAddr(), false);
     answer.nodes4 = buckets4.findClosestNodes(hash, now, TARGET_NODES);
     answer.nodes6 = buckets6.findClosestNodes(hash, now, TARGET_NODES);
     if (st != store.end() && not st->second.empty()) {
@@ -2237,7 +2156,7 @@ Dht::onGetValues(Sp<Node> node, const InfoHash& hash, want_t, const Query& query
 }
 
 void Dht::onGetValuesDone(const Sp<Node>& node,
-        net::NetworkEngine::RequestAnswer& a,
+        net::RequestAnswer& a,
         Sp<Search>& sr,
         const Sp<Query>& orig_query)
 {
@@ -2247,7 +2166,7 @@ void Dht::onGetValuesDone(const Sp<Node>& node,
     }
 
     DHT_LOG.d(sr->id, "[search %s] [node %s] got reply to 'get' with %u nodes",
-            sr->id.toString().c_str(), node->toString().c_str(), a.nodes4.size());
+            sr->id.toString().c_str(), node->toString().c_str(), a.nodes4.size()+a.nodes6.size());
 
     if (not a.ntoken.empty()) {
         if (not a.values.empty() or not a.fields.empty()) {
@@ -2308,7 +2227,7 @@ void Dht::onGetValuesDone(const Sp<Node>& node,
     }
 }
 
-net::NetworkEngine::RequestAnswer
+net::RequestAnswer
 Dht::onListen(Sp<Node> node, const InfoHash& hash, const Blob& token, size_t socket_id, const Query& query)
 {
     if (hash == zeroes) {
@@ -2329,7 +2248,7 @@ Dht::onListen(Sp<Node> node, const InfoHash& hash, const Blob& token, size_t soc
 
 void
 Dht::onListenDone(const Sp<Node>& node,
-        net::NetworkEngine::RequestAnswer& answer,
+        net::RequestAnswer& answer,
         Sp<Search>& sr)
 {
     DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got listen confirmation",
@@ -2342,30 +2261,31 @@ Dht::onListenDone(const Sp<Node>& node,
     }
 }
 
-net::NetworkEngine::RequestAnswer
-Dht::onAnnounce(Sp<Node> node,
+net::RequestAnswer
+Dht::onAnnounce(Sp<Node> n,
         const InfoHash& hash,
         const Blob& token,
         const std::vector<Sp<Value>>& values,
         const time_point& creation_date)
 {
+    auto& node = *n;
     if (hash == zeroes) {
-        DHT_LOG.w(node->id, "put with no info_hash");
+        DHT_LOG.w(node.id, "put with no info_hash");
         throw net::DhtProtocolException {
             net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION,
             net::DhtProtocolException::PUT_NO_INFOHASH
         };
     }
-    if (not tokenMatch(token, node->addr)) {
-        DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'put'", node->toString().c_str(), hash.toString().c_str());
+    if (not tokenMatch(token, node.getAddr())) {
+        DHT_LOG.w(hash, node.id, "[node %s] incorrect token %s for 'put'", node.toString().c_str(), hash.toString().c_str());
         throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::PUT_WRONG_TOKEN};
     }
     {
         // We store a value only if we think we're part of the
         // SEARCH_NODES nodes around the target id.
-        auto closest_nodes = buckets(node->getFamily()).findClosestNodes(hash, scheduler.time(), SEARCH_NODES);
+        auto closest_nodes = buckets(node.getFamily()).findClosestNodes(hash, scheduler.time(), SEARCH_NODES);
         if (closest_nodes.size() >= TARGET_NODES and hash.xorCmp(closest_nodes.back()->id, getNodeId()) < 0) {
-            DHT_LOG.w(hash, node->id, "[node %s] announce too far from the target. Dropping value.", node->toString().c_str());
+            DHT_LOG.w(hash, node.id, "[node %s] announce too far from the target. Dropping value.", node.toString().c_str());
             return {};
         }
     }
@@ -2373,7 +2293,7 @@ Dht::onAnnounce(Sp<Node> node,
     auto created = std::min(creation_date, scheduler.time());
     for (const auto& v : values) {
         if (v->id == Value::INVALID_ID) {
-            DHT_LOG.w(hash, node->id, "[value %s] incorrect value id", hash.toString().c_str());
+            DHT_LOG.w(hash, node.id, "[value %s] incorrect value id", hash.toString().c_str());
             throw net::DhtProtocolException {
                 net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION,
                 net::DhtProtocolException::PUT_INVALID_ID
@@ -2383,26 +2303,26 @@ Dht::onAnnounce(Sp<Node> node,
         Sp<Value> vc = v;
         if (lv) {
             if (*lv == *vc) {
-                DHT_LOG.w(hash, node->id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str());
+                DHT_LOG.d(hash, node.id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str());
             } else {
                 const auto& type = getType(lv->type);
-                if (type.editPolicy(hash, lv, vc, node->id, (sockaddr*)&node->addr.first, node->addr.second)) {
-                    DHT_LOG.d(hash, node->id, "[store %s] editing %s",
+                if (type.editPolicy(hash, lv, vc, node.id, node.getAddr())) {
+                    DHT_LOG.d(hash, node.id, "[store %s] editing %s",
                             hash.toString().c_str(), vc->toString().c_str());
-                    storageStore(hash, vc, created, &node->addr);
+                    storageStore(hash, vc, created, node.getAddr());
                 } else {
-                    DHT_LOG.d(hash, node->id, "[store %s] rejecting edition of %s because of storage policy",
+                    DHT_LOG.d(hash, node.id, "[store %s] rejecting edition of %s because of storage policy",
                             hash.toString().c_str(), vc->toString().c_str());
                 }
             }
         } else {
             // Allow the value to be edited by the storage policy
             const auto& type = getType(vc->type);
-            if (type.storePolicy(hash, vc, node->id, (sockaddr*)&node->addr.first, node->addr.second)) {
-                DHT_LOG.d(hash, node->id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str());
-                storageStore(hash, vc, created, &node->addr);
+            if (type.storePolicy(hash, vc, node.id, node.getAddr())) {
+                DHT_LOG.d(hash, node.id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str());
+                storageStore(hash, vc, created, node.getAddr());
             } else {
-                DHT_LOG.d(hash, node->id, "[store %s] rejecting storage of %s",
+                DHT_LOG.d(hash, node.id, "[store %s] rejecting storage of %s",
                         hash.toString().c_str(), vc->toString().c_str());
             }
         }
@@ -2410,13 +2330,13 @@ Dht::onAnnounce(Sp<Node> node,
     return {};
 }
 
-net::NetworkEngine::RequestAnswer
+net::RequestAnswer
 Dht::onRefresh(Sp<Node> node, const InfoHash& hash, const Blob& token, const Value::Id& vid)
 {
     using namespace net;
 
     const auto& now = scheduler.time();
-    if (not tokenMatch(token, node->addr)) {
+    if (not tokenMatch(token, node->getAddr())) {
         DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'put'", node->toString().c_str(), hash.toString().c_str());
         throw DhtProtocolException {DhtProtocolException::UNAUTHORIZED, DhtProtocolException::PUT_WRONG_TOKEN};
     }
@@ -2433,7 +2353,7 @@ Dht::onRefresh(Sp<Node> node, const InfoHash& hash, const Blob& token, const Val
 }
 
 void
-Dht::onAnnounceDone(const Sp<Node>& node, net::NetworkEngine::RequestAnswer& answer, Sp<Search>& sr)
+Dht::onAnnounceDone(const Sp<Node>& node, net::RequestAnswer& answer, Sp<Search>& sr)
 {
     DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got reply to put!",
             sr->id.toString().c_str(), node->toString().c_str());

src/dhtrunner.cpp

@@ -243,7 +243,7 @@ DhtRunner::getPublicAddressStr(sa_family_t af)
 {
     auto addrs = getPublicAddress(af);
     std::vector<std::string> ret(addrs.size());
-    std::transform(addrs.begin(), addrs.end(), ret.begin(), dht::printAddr);
+    std::transform(addrs.begin(), addrs.end(), ret.begin(), [](const SockAddr& a) { return a.toString(); });
     return ret;
 }
 
@@ -608,7 +608,7 @@ void
 DhtRunner::ping(const SockAddr& addr, DoneCallback&& cb)
 {
     std::lock_guard<std::mutex> lck(storage_mtx);
-    pending_ops_prio.emplace([addr,cb=std::move(cb)](SecureDht& dht) mutable {
+    pending_ops_prio.emplace([addr,cb](SecureDht& dht) mutable {
         dht.pingNode(addr, std::move(cb));
     });
     uv_async_send(&uv_async_);

src/network_engine.cpp

@@ -22,6 +22,7 @@
 #include "request.h"
 #include "default_types.h"
 #include "log_enable.h"
+#include "parsed_message.h"
 
 #include <msgpack.hpp>
 
@@ -53,7 +54,6 @@ constexpr std::chrono::seconds NetworkEngine::RX_TIMEOUT;
 
 const std::string NetworkEngine::my_v {"RNG1"};
 const constexpr uint16_t TransId::INVALID;
-std::mt19937 NetworkEngine::rd_device {dht::crypto::random_device{}()};
 
 const TransPrefix TransPrefix::PING = {"pn"};
 const TransPrefix TransPrefix::FIND_NODE = {"fn"};
@@ -69,89 +69,6 @@ static const uint8_t v4prefix[16] = {
 
 constexpr unsigned SEND_NODES {8};
 
-#ifdef _WIN32
-
-static bool
-set_nonblocking(int fd, int nonblocking)
-{
-    unsigned long mode = !!nonblocking;
-    int rc = ioctlsocket(fd, FIONBIO, &mode);
-    return rc == 0;
-}
-
-extern const char *inet_ntop(int, const void *, char *, socklen_t);
-
-#else
-
-static bool
-set_nonblocking(int fd, int nonblocking)
-{
-    int rc = fcntl(fd, F_GETFL, 0);
-    if (rc < 0)
-        return false;
-    rc = fcntl(fd, F_SETFL, nonblocking?(rc | O_NONBLOCK):(rc & ~O_NONBLOCK));
-    return rc >= 0;
-}
-
-#endif
-
-enum class MessageType {
-    Error = 0,
-    Reply,
-    Ping,
-    FindNode,
-    GetValues,
-    AnnounceValue,
-    Refresh,
-    Listen,
-    ValueData
-};
-
-struct ParsedMessage {
-    MessageType type;
-    /* Node ID of the sender */
-    InfoHash id;
-    /* Network id */
-    NetId network {0};
-    /* hash for which values are requested */
-    InfoHash info_hash;
-    /* target id around which to find nodes */
-    InfoHash target;
-    /* transaction id */
-    TransId tid;
-    /* tid for packets going through request socket */
-    TransId socket_id;
-    /* security token */
-    Blob token;
-    /* the value id (announce confirmation) */
-    Value::Id value_id;
-    /* time when value was first created */
-    time_point created { time_point::max() };
-    /* IPv4 nodes in response to a 'find' request */
-    Blob nodes4_raw, nodes6_raw;
-    std::vector<Sp<Node>> nodes4, nodes6;
-    /* values to store or retreive request */
-    std::vector<Sp<Value>> values;
-    /* index for fields values */
-    std::vector<Sp<FieldValueIndex>> fields;
-    /** When part of the message header: {index -> (total size, {})}
-     *  When part of partial value data: {index -> (offset, part_data)} */
-    std::map<unsigned, std::pair<unsigned, Blob>> value_parts;
-    /* query describing a filter to apply on values. */
-    Query query;
-    /* states if ipv4 or ipv6 request */
-    want_t want;
-    /* error code in case of error */
-    uint16_t error_code;
-    /* reported address by the distant node */
-    std::string ua;
-    SockAddr addr;
-    void msgpack_unpack(msgpack::object o);
-
-    bool append(const ParsedMessage& block);
-    bool complete();
-};
-
 struct NetworkEngine::PartialMessage {
     SockAddr from;
     time_point start;
@@ -169,7 +86,7 @@ serializeValues(const std::vector<Sp<Value>>& st)
     return svals;
 }
 
-NetworkEngine::RequestAnswer::RequestAnswer(ParsedMessage&& msg)
+RequestAnswer::RequestAnswer(ParsedMessage&& msg)
  : ntoken(std::move(msg.token)), values(std::move(msg.values)), fields(std::move(msg.fields)),
     nodes4(std::move(msg.nodes4)), nodes6(std::move(msg.nodes6)) {}
 
@@ -189,7 +106,6 @@ NetworkEngine::NetworkEngine(uv_loop_t* loop, const NetworkConfig& config, NetId
     network(net), DHT_LOG(log), sock(std::make_shared<UdpSocket>(loop)), tcp_sock(std::make_shared<TcpSocket>(loop)),
     id_key(crypto::EcSecretKey::generate()), scheduler(scheduler)
 {
-    transaction_id = std::uniform_int_distribution<decltype(transaction_id)>{1}(rd_device);
     using namespace std::placeholders;
     sock->open(config.bind_addr.c_str(), config.bind_port, std::bind(&NetworkEngine::processMessage, this, _1, _2, _3));
     tcp_sock->listen(config.bind_addr.c_str(), config.bind_port, [&](Sp<TcpSocket>&& new_socket){
@@ -208,7 +124,7 @@ NetworkEngine::tellListener(Sp<Node> node, uint32_t socket_id, const InfoHash& h
 {
     auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6);
     try {
-        sendNodesValues(node, TransId((char*)&socket_id, 4), nnodes.first, nnodes.second, values, query, ntoken);
+        sendNodesValues(node, TransId(socket_id), nnodes.first, nnodes.second, values, query, ntoken);
     } catch (const std::overflow_error& e) {
         DHT_LOG.e("Can't send value: buffer not large enough !");
     }
@@ -220,40 +136,9 @@ NetworkEngine::isRunning(sa_family_t /*af*/) const
     return sock->isRunning();
 }
 
-Sp<Socket>
-NetworkEngine::openSocket(const Sp<Node>& node, TransPrefix tp, SocketCb&& cb)
-{
-    auto tid = TransId {tp, getNewTid()};
-    auto s = opened_sockets.emplace(tid, std::make_shared<Socket>(node, tid, cb));
-    if (not s.second)
-        DHT_LOG.e(node->id, "[node %s] socket (tid: %d) already opened!", node->id.toString().c_str(), tid.toInt());
-    return s.first->second;
-}
-
-void
-NetworkEngine::closeSocket(Sp<Socket> socket)
-{
-    if (socket)
-        opened_sockets.erase(socket->id);
-}
-
-void
-NetworkEngine::cancelRequest(Sp<Request>& req)
-{
-    if (req) {
-        req->cancel();
-        closeSocket(req->socket);
-        requests.erase(req->tid);
-    }
-}
-
 void
 NetworkEngine::clear()
-{
-    for (auto& req : requests)
-        req.second->setExpired();
-    requests.clear();
-}
+{}
 
 void
 NetworkEngine::connectivityChanged(sa_family_t af)
@@ -265,18 +150,16 @@ void
 NetworkEngine::requestStep(Sp<Request> sreq)
 {
     auto& req = *sreq;
-    if (not req.pending()) {
-        if (req.cancelled())
-            requests.erase(req.tid);
+    if (not req.pending())
         return;
-    }
 
     auto now = scheduler.time();
     auto& node = *req.node;
     if (req.isExpired(now)) {
         DHT_LOG.e(node.id, "[node %s] expired !", node.toString().c_str());
         node.setExpired();
-        requests.erase(req.tid);
+        if (node.id == zeroes)
+            requests.erase(req.tid);
         return;
     } else if (req.attempt_count == 1) {
         req.on_expired(req, false);
@@ -293,29 +176,30 @@ NetworkEngine::requestStep(Sp<Request> sreq)
                 sreq->node->setExpired();
             }
         });
-    } else {
-        // UDP
-        auto err = send(req.msg,
-                (node.reply_time >= now - UDP_REPLY_TIME) ? 0 : MSG_CONFIRM,
-                node);
-        if (err == ENETUNREACH  ||
-            err == EHOSTUNREACH ||
-            err == EAFNOSUPPORT)
-        {
-            node.setExpired();
-            requests.erase(req.tid);
-        } else {
-            if (err != EAGAIN) {
-                ++req.attempt_count;
-            }
-            req.last_try = now;
-            std::weak_ptr<Request> wreq = sreq;
-            scheduler.add(req.last_try + Node::MAX_RESPONSE_TIME, [this,wreq] {
-                if (auto req = wreq.lock())
-                    requestStep(req);
-            });
-        }
+        return;
     }
+    // UDP
+    std::weak_ptr<Request> wreq = sreq;
+    send(req.msg, 0, node, [this,wreq](int status) {
+        if (auto req = wreq.lock()) {
+            if (status >= 0) {
+                ++req->attempt_count;
+                return;
+            }
+            if (status == -EAGAIN)
+                return;
+            auto& node = *req->node;
+            DHT_LOG.e(node.id, "[node %s] can't send packet: %d %s", node.toString().c_str(), status, uv_strerror(status));
+            node.setExpired();
+            if (node.id == zeroes)
+                requests.erase(req->tid);
+        }
+    });
+    req.last_try = now;
+    scheduler.add(req.last_try + Node::MAX_RESPONSE_TIME, [this,wreq] {
+        if (auto req = wreq.lock())
+            requestStep(req);
+    });
 }
 
 /**
@@ -323,18 +207,13 @@ NetworkEngine::requestStep(Sp<Request> sreq)
  * be made before the request expires.
  */
 void
-NetworkEngine::sendRequest(Sp<Request>& request)
+NetworkEngine::sendRequest(const Sp<Request>& request)
 {
+    auto& node = request->node;
+    if (node->id == zeroes)
+        requests.emplace(request->tid, request);
     request->start = scheduler.time();
-    auto e = requests.emplace(request->tid, request);
-    if (not e.second) {
-        // Should not happen !
-        // Try to handle this scenario as well as we can
-        e.first->second->setExpired();
-        e.first->second = request;
-        DHT_LOG.e(request->node->id, "Request already existed (tid: %d)!", request->tid.getTid());
-    }
-    request->node->requested(request);
+    node->requested(request);
     requestStep(request);
 }
 
@@ -345,15 +224,15 @@ NetworkEngine::rateLimit(const SockAddr& addr)
 {
     const auto& now = scheduler.time();
 
-    // occasional IP limiter maintenance
-    std::bernoulli_distribution rand_trial(1./128.);
-    if (rand_trial(rd_device)) {
+    // occasional IP limiter maintenance (a few times every second at max rate)
+    if (limiter_maintenance++ == MAX_REQUESTS_PER_SEC/8) {
         for (auto it = address_rate_limiter.begin(); it != address_rate_limiter.end();) {
             if (it->second.maintain(now) == 0)
                 address_rate_limiter.erase(it++);
             else
                 ++it;
         }
+        limiter_maintenance = 0;
     }
 
     auto it = address_rate_limiter.emplace(addr, IpLimiter{});
@@ -374,7 +253,7 @@ NetworkEngine::isMartian(const SockAddr& addr)
               ((address[0] & 0xE0) == 0xE0);
     }
     case AF_INET6: {
-        if (addr.second < sizeof(sockaddr_in6))
+        if (addr.getLength() < sizeof(sockaddr_in6))
             return true;
         const auto& sin6 = addr.getIPv6();
         const uint8_t* address = (const uint8_t*)&sin6.sin6_addr;
@@ -394,15 +273,7 @@ void
 NetworkEngine::blacklistNode(const Sp<Node>& n)
 {
     n->setExpired();
-    for (auto rit = requests.begin(); rit != requests.end();) {
-        if (rit->second->node == n) {
-            //rit->second->cancel();
-            requests.erase(rit++);
-        } else {
-            ++rit;
-        }
-    }
-    blacklist.emplace(n->addr);
+    blacklist.emplace(n->getAddr());
 }
 
 bool
@@ -600,44 +471,37 @@ Sp<Node>
 NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& from, const Sp<TcpSocket>& tcpsock)
 {
     const auto& now = scheduler.time();
-    Sp<Node> node;
+    auto node = cache.getNode(msg->id, from, tcpsock, now, true, msg->is_client);
 
     if (msg->type == MessageType::Error or msg->type == MessageType::Reply) {
+        auto rsocket = node->getSocket(msg->tid.toInt());
+        auto req = node->getRequest(msg->tid);
+
         /* either response for a request or data for an opened socket */
-        auto req_it = requests.find(msg->tid);
-        auto rsocket_it = opened_sockets.end();
-        if (req_it == requests.end())
-            rsocket_it = opened_sockets.find(msg->tid);
-        if (req_it == requests.end() and rsocket_it == opened_sockets.end()) {
-            DHT_LOG.e("Can't find transaction.");
-            return {};
-        }
-
-        auto req = req_it != requests.end() ? req_it->second : nullptr;
-        auto rsocket = rsocket_it != opened_sockets.end() ? rsocket_it->second : nullptr;
-
-        node = req ? req->node : rsocket->node;
-        if (node->id != msg->id) {
-            if (node->id == zeroes) // received reply to a message sent when we didn't know the node ID.
-                node = cache.getNode(msg->id, from, tcpsock, now, true);
-            else {
-                // received reply from unexpected node
+        if (not req and not rsocket) {
+            auto req_it = requests.find(msg->tid);
+            if (req_it != requests.end() and req_it->second->node->id == zeroes) {
+                req = req_it->second;
+                req->node = node;
+                requests.erase(req_it);
+            } else {
                 node->received(now, req);
-                onNewNode(node, 2);
-                DHT_LOG.w(node->id, "[node %s] message received from unexpected node", node->toString().c_str());
+                if (not node->isClient())
+                    onNewNode(node, 1);
+                DHT_LOG.e(node->id, "[node %s] Can't find transaction", node->toString().c_str());
                 return node;
             }
-        } else
-            node->update(from, tcpsock);
+        }
+        node->update(from, tcpsock);
 
         node->received(now, req);
 
-        onNewNode(node, 2);
+        if (not node->isClient())
+            onNewNode(node, 2);
         onReportedAddr(msg->id, msg->addr);
 
         if (req and (req->cancelled() or req->expired() or req->completed())) {
             DHT_LOG.w(node->id, "[node %s] response to expired, cancelled or completed request", node->toString().c_str());
-            requests.erase(req_it);
             return {};
         }
 
@@ -658,28 +522,26 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
             break;
         }
         case MessageType::Reply:
-         if (req) { /* request reply */
-             if (msg->type == MessageType::AnnounceValue or msg->type == MessageType::Listen)
-                 req->node->authSuccess();
+            if (req) { /* request reply */
+                if (msg->type == MessageType::AnnounceValue or msg->type == MessageType::Listen)
+                    req->node->authSuccess();
+                req->reply_time = scheduler.time();
 
-             // erase before calling callback to make sure iterator is still valid
-             requests.erase(req_it);
-             req->reply_time = scheduler.time();
-
-             deserializeNodes(*msg, from);
-             req->setDone(std::move(*msg));
-             break;
-         } else { /* request socket data */
-             deserializeNodes(*msg, from);
-             rsocket->on_receive(rsocket->node, std::move(*msg));
-         }
+                deserializeNodes(*msg, from);
+                req->setDone(std::move(*msg));
+                break;
+            } else { /* request socket data */
+                deserializeNodes(*msg, from);
+                rsocket->on_receive(node, std::move(*msg));
+            }
+            break;
         default:
             break;
         }
     } else {
-        node = cache.getNode(msg->id, from, tcpsock, now, true);
         node->received(now, {});
-        onNewNode(node, 1);
+        if (not node->isClient())
+            onNewNode(node, 1);
         try {
             switch (msg->type) {
             case MessageType::Ping:
@@ -726,9 +588,7 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
             case MessageType::Listen: {
                 DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                 ++in_stats.listen;
-                /* TODO: backward compatibility check to remove in a few versions (see ::sendListen doc) */
-                auto socket_id = msg->socket_id.getTid() ? msg->socket_id : TransId { TransPrefix::GET_VALUES, msg->tid.getTid() };
-                RequestAnswer answer = onListen(node, msg->info_hash, msg->token, socket_id.toInt(), std::move(msg->query));
+                RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id.toInt(), std::move(msg->query));
                 sendListenConfirmation(node, msg->tid);
                 break;
             }
@@ -756,10 +616,10 @@ packToken(msgpack::packer<msgpack::sbuffer>& pk, const Blob& token)
 void
 insertAddr(msgpack::packer<msgpack::sbuffer>& pk, const SockAddr& addr)
 {
-    size_t addr_len = std::min<size_t>(addr.second,
+    size_t addr_len = std::min<size_t>(addr.getLength(),
                      (addr.getFamily() == AF_INET) ? sizeof(in_addr) : sizeof(in6_addr));
-    void* addr_ptr = (addr.getFamily() == AF_INET) ? (void*)&((sockaddr_in*)&addr.first)->sin_addr
-                                                : (void*)&((sockaddr_in6*)&addr.first)->sin6_addr;
+    void* addr_ptr = (addr.getFamily() == AF_INET) ? (void*)&addr.getIPv4().sin_addr
+                                                : (void*)&addr.getIPv6().sin6_addr;
     pk.pack("sa");
     pk.pack_bin(addr_len);
     pk.pack_bin_body((char*)addr_ptr, addr_len);
@@ -809,12 +669,12 @@ NetworkEngine::send(msgpack::sbuffer& msg, const Node& node, const Sp<TcpSocket>
         return node.sock->write(data, size);
     } else {
         DHT_LOG.d(node.id, "[node %s] send (datagram) %lu", node.getId().to_c_str(), msg.size());
-        return sock->send(data, size, node.addr);
+        return sock->send(data, size, node.getAddr());
     }
 }
 
 int
-NetworkEngine::send(const Blob& msg, int /*flags*/, const Node& node)
+NetworkEngine::send(const Blob& msg, int /*flags*/, const Node& node, UdpSocket::OnSent&& cb)
 {
     DHT_LOG.d(node.id, "[node %s] send %lu", node.getId().to_c_str(), msg.size());
     uint8_t* data;
@@ -848,13 +708,13 @@ NetworkEngine::send(const Blob& msg, int /*flags*/, const Node& node)
         return node.sock->write(data, size);
     } else {
         DHT_LOG.d(node.id, "[node %s] send (datagram) %lu", node.getId().to_c_str(), msg.size());
-        return sock->send(data, size, node.addr);
+        return sock->send(data, size, node.getAddr(), std::move(cb));
     }
 }
 
 Sp<Request>
 NetworkEngine::sendPing(const Sp<Node>& node, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
-    auto tid = TransId {TransPrefix::PING, getNewTid()};
+    auto tid = TransId {TransPrefix::PING, node->getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(5+(network?1:0));
@@ -898,7 +758,7 @@ NetworkEngine::sendPong(const Sp<Node>& node, TransId tid) {
 
     pk.pack(std::string("r")); pk.pack_map(2);
       pk.pack(std::string("id")); pk.pack(getNodeId());
-      insertAddr(pk, node->addr);
+      insertAddr(pk, node->getAddress());
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
@@ -914,7 +774,7 @@ NetworkEngine::sendPong(const Sp<Node>& node, TransId tid) {
 Sp<Request>
 NetworkEngine::sendFindNode(const Sp<Node>& n, const InfoHash& target, want_t want,
         RequestCb&& on_done, RequestExpiredCb&& on_expired) {
-    auto tid = TransId {TransPrefix::FIND_NODE, getNewTid()};
+    auto tid = TransId {TransPrefix::FIND_NODE, n->getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(5+(network?1:0));
@@ -960,7 +820,7 @@ NetworkEngine::sendFindNode(const Sp<Node>& n, const InfoHash& target, want_t wa
 Sp<Request>
 NetworkEngine::sendGetValues(const Sp<Node>& n, const InfoHash& info_hash, const Query& query, want_t want,
         RequestCb&& on_done, RequestExpiredCb&& on_expired) {
-    auto tid = TransId {TransPrefix::GET_VALUES, getNewTid()};
+    auto tid = TransId {TransPrefix::GET_VALUES, n->getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(5+(network?1:0));
@@ -1007,20 +867,18 @@ NetworkEngine::sendGetValues(const Sp<Node>& n, const InfoHash& info_hash, const
 
 SockAddr deserializeIPv4(const uint8_t* ni) {
     SockAddr addr;
-    auto& sin = addr.getIPv4();
-    std::fill_n((uint8_t*)&sin, sizeof(sockaddr_in), 0);
     addr.setFamily(AF_INET);
-    memcpy(&sin.sin_addr, ni, 4);
-    memcpy(&sin.sin_port, ni + 4, 2);
+    auto& sin = addr.getIPv4();
+    std::memcpy(&sin.sin_addr, ni, 4);
+    std::memcpy(&sin.sin_port, ni + 4, 2);
     return addr;
 }
 SockAddr deserializeIPv6(const uint8_t* ni) {
     SockAddr addr;
-    auto& sin6 = addr.getIPv6();
-    std::fill_n((uint8_t*)&sin6, sizeof(sockaddr_in6), 0);
     addr.setFamily(AF_INET6);
-    memcpy(&sin6.sin6_addr, ni, 16);
-    memcpy(&sin6.sin6_port, ni + 16, 2);
+    auto& sin6 = addr.getIPv6();
+    std::memcpy(&sin6.sin6_addr, ni, 16);
+    std::memcpy(&sin6.sin6_port, ni + 16, 2);
     return addr;
 }
 
@@ -1028,41 +886,40 @@ void
 NetworkEngine::deserializeNodes(ParsedMessage& msg, const SockAddr& from) {
     if (msg.nodes4_raw.size() % NODE4_INFO_BUF_LEN != 0 || msg.nodes6_raw.size() % NODE6_INFO_BUF_LEN != 0) {
         throw DhtProtocolException {DhtProtocolException::WRONG_NODE_INFO_BUF_LEN};
-    } else {
-        // deserialize nodes
-        const auto& now = scheduler.time();
-        for (unsigned i = 0; i < msg.nodes4_raw.size() / NODE4_INFO_BUF_LEN; i++) {
-            uint8_t *ni = msg.nodes4_raw.data() + i * NODE4_INFO_BUF_LEN;
-            const InfoHash& ni_id = *reinterpret_cast<InfoHash*>(ni);
-            if (ni_id == getNodeId())
-                continue;
-            SockAddr addr = deserializeIPv4(ni + ni_id.size());
-            if (addr.isLoopback() and from.getFamily() == AF_INET) {
-                auto port = addr.getPort();
-                addr = from;
-                addr.setPort(port);
-            }
-            if (isMartian(addr) || isNodeBlacklisted(addr))
-                continue;
-            msg.nodes4.emplace_back(cache.getNode(ni_id, addr, {}, now, false));
-            onNewNode(msg.nodes4.back(), 0);
+    }
+    // deserialize nodes
+    const auto& now = scheduler.time();
+    for (unsigned i = 0, n = msg.nodes4_raw.size() / NODE4_INFO_BUF_LEN; i < n; i++) {
+        const uint8_t* ni = msg.nodes4_raw.data() + i * NODE4_INFO_BUF_LEN;
+        const auto& ni_id = *reinterpret_cast<const InfoHash*>(ni);
+        if (ni_id == getNodeId())
+            continue;
+        SockAddr addr = deserializeIPv4(ni + ni_id.size());
+        if (addr.isLoopback() and from.getFamily() == AF_INET) {
+            auto port = addr.getPort();
+            addr = from;
+            addr.setPort(port);
         }
-        for (unsigned i = 0; i < msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i++) {
-            uint8_t *ni = msg.nodes6_raw.data() + i * NODE6_INFO_BUF_LEN;
-            const InfoHash& ni_id = *reinterpret_cast<InfoHash*>(ni);
-            if (ni_id == getNodeId())
-                continue;
-            SockAddr addr = deserializeIPv6(ni + ni_id.size());
-            if (addr.isLoopback() and from.getFamily() == AF_INET6) {
-                auto port = addr.getPort();
-                addr = from;
-                addr.setPort(port);
-            }
-            if (isMartian(addr) || isNodeBlacklisted(addr))
-                continue;
-            msg.nodes6.emplace_back(cache.getNode(ni_id, addr, {}, now, false));
-            onNewNode(msg.nodes6.back(), 0);
+        if (isMartian(addr) || isNodeBlacklisted(addr))
+            continue;
+        msg.nodes4.emplace_back(cache.getNode(ni_id, addr, {}, now, false));
+        onNewNode(msg.nodes4.back(), 0);
+    }
+    for (unsigned i = 0, n = msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i < n; i++) {
+        const uint8_t* ni = msg.nodes6_raw.data() + i * NODE6_INFO_BUF_LEN;
+        const auto& ni_id = *reinterpret_cast<const InfoHash*>(ni);
+        if (ni_id == getNodeId())
+            continue;
+        SockAddr addr = deserializeIPv6(ni + ni_id.size());
+        if (addr.isLoopback() and from.getFamily() == AF_INET6) {
+            auto port = addr.getPort();
+            addr = from;
+            addr.setPort(port);
         }
+        if (isMartian(addr) || isNodeBlacklisted(addr))
+            continue;
+        msg.nodes6.emplace_back(cache.getNode(ni_id, addr, {}, now, false));
+        onNewNode(msg.nodes6.back(), 0);
     }
 }
 
@@ -1131,7 +988,7 @@ NetworkEngine::sendNodesValues(const Sp<Node>& node, TransId tid, const Blob& no
     pk.pack(std::string("r"));
     pk.pack_map(2 + (not st.empty()?1:0) + (nodes.size()>0?1:0) + (nodes6.size()>0?1:0) + (not token.empty()?1:0));
     pk.pack(std::string("id")); pk.pack(getNodeId());
-    insertAddr(pk, node->addr);
+    insertAddr(pk, node->getAddress());
     if (nodes.size() > 0) {
         pk.pack(std::string("n4"));
         pk.pack_bin(nodes.size());
@@ -1175,7 +1032,7 @@ NetworkEngine::sendNodesValues(const Sp<Node>& node, TransId tid, const Blob& no
 
     // send parts
     if (not svals.empty())
-        sendValueParts(tid, svals, node->addr);
+        sendValueParts(tid, svals, node->getAddress());
 }
 
 Blob
@@ -1190,21 +1047,21 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, std::vector<Sp<No
         bnodes.resize(NODE4_INFO_BUF_LEN * nnode);
         for (size_t i=0; i<nnode; i++) {
             const Node& n = *nodes[i];
-            sockaddr_in *sin = (sockaddr_in*)&n.addr.first;
+            const auto& sin = n.getAddr().getIPv4();
             auto dest = bnodes.data() + NODE4_INFO_BUF_LEN * i;
             memcpy(dest, n.id.data(), HASH_LEN);
-            memcpy(dest + HASH_LEN, &sin->sin_addr, sizeof(in_addr));
-            memcpy(dest + HASH_LEN + sizeof(in_addr), &sin->sin_port, sizeof(in_port_t));
+            memcpy(dest + HASH_LEN, &sin.sin_addr, sizeof(in_addr));
+            memcpy(dest + HASH_LEN + sizeof(in_addr), &sin.sin_port, sizeof(in_port_t));
         }
     } else if (af == AF_INET6) {
         bnodes.resize(NODE6_INFO_BUF_LEN * nnode);
         for (size_t i=0; i<nnode; i++) {
             const Node& n = *nodes[i];
-            sockaddr_in6 *sin6 = (sockaddr_in6*)&n.addr.first;
+            const auto& sin6 = n.getAddr().getIPv6();
             auto dest = bnodes.data() + NODE6_INFO_BUF_LEN * i;
             memcpy(dest, n.id.data(), HASH_LEN);
-            memcpy(dest + HASH_LEN, &sin6->sin6_addr, sizeof(in6_addr));
-            memcpy(dest + HASH_LEN + sizeof(in6_addr), &sin6->sin6_port, sizeof(in_port_t));
+            memcpy(dest + HASH_LEN, &sin6.sin6_addr, sizeof(in6_addr));
+            memcpy(dest + HASH_LEN + sizeof(in6_addr), &sin6.sin6_port, sizeof(in_port_t));
         }
     }
     return bnodes;
@@ -1241,31 +1098,27 @@ NetworkEngine::sendListen(const Sp<Node>& n,
     //n->startTcp(scheduler.getLoop());
     if (not n->sock) {
         n->sock = std::make_shared<TcpSocket>(scheduler.getLoop());
-        n->sock->connect(n->addr.get(), [this,n](int status){
+        n->sock->connect(n->getAddr().get(), [this,n](int status){
             if (status == 0)
                 startTcp(n->sock, n);
         });
     }
 
-    Sp<Socket> socket;
-    auto tid = TransId { TransPrefix::LISTEN, previous ? previous->tid.getTid() : getNewTid() };
+    uint32_t socket;
+    auto tid = TransId { TransPrefix::LISTEN, n->getNewTid() };
     if (previous and previous->node == n) {
-        socket = previous->socket;
+        socket = previous->getSocket();
     } else {
         if (previous)
             DHT_LOG.e(hash, "[node %s] trying refresh listen contract with wrong node", previous->node->toString().c_str());
-        /* TODO: Manually creating a socket for "listen" backward compatibility.
-         * As soon as this is not an issue anymore, switch to using
-         * ::createSocket function! */
-        auto sid = TransId {TransPrefix::GET_VALUES, tid.getTid()};
-        socket = std::make_shared<Socket>(n, sid, socket_cb);
-        opened_sockets.emplace(sid, socket);
+        socket = n->openSocket(std::move(socket_cb));
     }
 
     if (not socket) {
         DHT_LOG.e(hash, "[node %s] unable to get a valid socket for listen. Aborting listen", n->toString().c_str());
         return {};
     }
+    TransId sid(socket);
 
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
@@ -1276,8 +1129,8 @@ NetworkEngine::sendListen(const Sp<Node>& n,
       pk.pack(std::string("id"));    pk.pack(getNodeId());
       pk.pack(std::string("h"));     pk.pack(hash);
       pk.pack(std::string("token")); packToken(pk, token);
-      pk.pack(std::string("sid"));  pk.pack_bin(socket->id.size());
-                                     pk.pack_bin_body((const char*)socket->id.data(), socket->id.size());
+      pk.pack(std::string("sid"));  pk.pack_bin(sid.size());
+                                     pk.pack_bin_body((const char*)sid.data(), sid.size());
       if (has_query) {
           pk.pack(std::string("q")); pk.pack(query);
       }
@@ -1316,7 +1169,7 @@ NetworkEngine::sendListenConfirmation(const Sp<Node>& node, TransId tid) {
 
     pk.pack(std::string("r")); pk.pack_map(2);
       pk.pack(std::string("id")); pk.pack(getNodeId());
-      insertAddr(pk, node->addr);
+      insertAddr(pk, node->getAddress());
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
@@ -1338,7 +1191,7 @@ NetworkEngine::sendAnnounceValue(const Sp<Node>& n,
         RequestCb&& on_done,
         RequestExpiredCb&& on_expired)
 {
-    auto tid = TransId {TransPrefix::ANNOUNCE_VALUES, getNewTid()};
+    auto tid = TransId {TransPrefix::ANNOUNCE_VALUES, n->getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(5+(network?1:0));
@@ -1383,7 +1236,7 @@ NetworkEngine::sendAnnounceValue(const Sp<Node>& n,
     );
     sendRequest(req);
     if (not v.empty())
-        sendValueParts(tid, v, n->addr);
+        sendValueParts(tid, v, n->getAddr());
     ++out_stats.put;
     return req;
 }
@@ -1396,7 +1249,7 @@ NetworkEngine::sendRefreshValue(const Sp<Node>& n,
                 RequestCb&& on_done,
                 RequestExpiredCb&& on_expired)
 {
-    auto tid = TransId {TransPrefix::REFRESH, getNewTid()};
+    auto tid = TransId {TransPrefix::REFRESH, n->getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(5+(network?1:0));
@@ -1449,7 +1302,7 @@ NetworkEngine::sendValueAnnounced(const Sp<Node>& node, TransId tid, Value::Id v
     pk.pack(std::string("r")); pk.pack_map(3);
       pk.pack(std::string("id"));  pk.pack(getNodeId());
       pk.pack(std::string("vid")); pk.pack(vid);
-      insertAddr(pk, node->addr);
+      insertAddr(pk, node->getAddress());
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
@@ -1494,187 +1347,6 @@ NetworkEngine::sendError(const Sp<Node>& node,
     send(buffer, *node, tcpsock);
 }
 
-void
-ParsedMessage::msgpack_unpack(msgpack::object msg)
-{
-    auto y = findMapValue(msg, "y");
-    auto r = findMapValue(msg, "r");
-    auto e = findMapValue(msg, "e");
-    auto v = findMapValue(msg, "p");
-
-    if (auto t = findMapValue(msg, "t"))
-        tid = {t->as<std::array<char, 4>>()};
-
-    if (auto rv = findMapValue(msg, "v"))
-        ua = rv->as<std::string>();
-
-    if (auto netid = findMapValue(msg, "n"))
-        network = netid->as<NetId>();
-
-    std::string q;
-    if (auto rq = findMapValue(msg, "q")) {
-        if (rq->type != msgpack::type::STR)
-            throw msgpack::type_error();
-        q = rq->as<std::string>();
-    }
-
-    if (e)
-        type = MessageType::Error;
-    else if (r)
-        type = MessageType::Reply;
-    else if (v)
-        type = MessageType::ValueData;
-    else if (y and y->as<std::string>() != "q")
-        throw msgpack::type_error();
-    else if (q == "ping")
-        type = MessageType::Ping;
-    else if (q == "find")
-        type = MessageType::FindNode;
-    else if (q == "get")
-        type = MessageType::GetValues;
-    else if (q == "listen")
-        type = MessageType::Listen;
-    else if (q == "put")
-        type = MessageType::AnnounceValue;
-    else if (q == "refresh")
-        type = MessageType::Refresh;
-    else
-        throw msgpack::type_error();
-
-    if (type == MessageType::ValueData) {
-        if (v->type != msgpack::type::MAP)
-            throw msgpack::type_error();
-        for (size_t i = 0; i < v->via.map.size; ++i) {
-            auto& vdat = v->via.map.ptr[i];
-            auto o = findMapValue(vdat.val, "o");
-            auto d = findMapValue(vdat.val, "d");
-            if (not o or not d)
-                continue;
-            value_parts.emplace(vdat.key.as<unsigned>(), std::pair<size_t, Blob>(o->as<size_t>(), unpackBlob(*d)));
-        }
-        return;
-    }
-
-    auto a = findMapValue(msg, "a");
-    if (!a && !r && !e)
-        throw msgpack::type_error();
-    auto& req = a ? *a : (r ? *r : *e);
-
-    if (e) {
-        if (e->type != msgpack::type::ARRAY)
-            throw msgpack::type_error();
-        error_code = e->via.array.ptr[0].as<uint16_t>();
-    }
-
-    if (auto t = findMapValue(req, "sid"))
-        socket_id = {t->as<std::array<char, 4>>()};
-
-    if (auto rid = findMapValue(req, "id"))
-        id = {*rid};
-
-    if (auto rh = findMapValue(req, "h"))
-        info_hash = {*rh};
-
-    if (auto rtarget = findMapValue(req, "target"))
-        target = {*rtarget};
-
-    if (auto rquery = findMapValue(req, "q"))
-        query.msgpack_unpack(*rquery);
-
-    if (auto otoken = findMapValue(req, "token"))
-        token = unpackBlob(*otoken);
-
-    if (auto vid = findMapValue(req, "vid"))
-        value_id = vid->as<Value::Id>();
-
-    if (auto rnodes4 = findMapValue(req, "n4"))
-        nodes4_raw = unpackBlob(*rnodes4);
-
-    if (auto rnodes6 = findMapValue(req, "n6"))
-        nodes6_raw = unpackBlob(*rnodes6);
-
-    if (auto sa = findMapValue(req, "sa")) {
-        if (sa->type != msgpack::type::BIN)
-            throw msgpack::type_error();
-        auto l = sa->via.bin.size;
-        if (l == sizeof(in_addr)) {
-            auto a = (sockaddr_in*)&addr.first;
-            std::fill_n((uint8_t*)a, sizeof(sockaddr_in), 0);
-            a->sin_family = AF_INET;
-            a->sin_port = 0;
-            std::copy_n(sa->via.bin.ptr, l, (char*)&a->sin_addr);
-            addr.second = sizeof(sockaddr_in);
-        } else if (l == sizeof(in6_addr)) {
-            auto a = (sockaddr_in6*)&addr.first;
-            std::fill_n((uint8_t*)a, sizeof(sockaddr_in6), 0);
-            a->sin6_family = AF_INET6;
-            a->sin6_port = 0;
-            std::copy_n(sa->via.bin.ptr, l, (char*)&a->sin6_addr);
-            addr.second = sizeof(sockaddr_in6);
-        }
-    } else
-        addr.second = 0;
-
-    if (auto rcreated = findMapValue(req, "c"))
-        created = from_time_t(rcreated->as<std::time_t>());
-
-    if (auto rvalues = findMapValue(req, "values")) {
-        if (rvalues->type != msgpack::type::ARRAY)
-            throw msgpack::type_error();
-        for (size_t i = 0; i < rvalues->via.array.size; i++) {
-            auto& packed_v = rvalues->via.array.ptr[i];
-            if (packed_v.type == msgpack::type::POSITIVE_INTEGER) {
-                // Skip oversize values with a small margin for header overhead
-                if (packed_v.via.u64 > MAX_VALUE_SIZE + 32)
-                    continue;
-                value_parts.emplace(i, std::make_pair(packed_v.via.u64, Blob{}));
-            } else {
-                try {
-                    values.emplace_back(std::make_shared<Value>(rvalues->via.array.ptr[i]));
-                } catch (const std::exception& e) {
-                    //DHT_LOG_WARN("Error reading value: %s", e.what());
-                }
-            }
-        }
-    } else if (auto raw_fields = findMapValue(req, "fields")) {
-        if (auto rfields = findMapValue(*raw_fields, "f")) {
-            auto vfields = rfields->as<std::set<Value::Field>>();
-            if (auto rvalues = findMapValue(*raw_fields, "v")) {
-                if (rvalues->type != msgpack::type::ARRAY)
-                    throw msgpack::type_error();
-                size_t val_num = rvalues->via.array.size / vfields.size();
-                for (size_t i = 0; i < val_num; ++i) {
-                    try {
-                        auto v = std::make_shared<FieldValueIndex>();
-                        v->msgpack_unpack_fields(vfields, *rvalues, i*vfields.size());
-                        fields.emplace_back(std::move(v));
-                    } catch (const std::exception& e) { }
-                }
-            }
-        } else {
-            throw msgpack::type_error();
-        }
-    }
-
-    if (auto w = findMapValue(req, "w")) {
-        if (w->type != msgpack::type::ARRAY)
-            throw msgpack::type_error();
-        want = 0;
-        for (unsigned i=0; i<w->via.array.size; i++) {
-            auto& val = w->via.array.ptr[i];
-            try {
-                auto w = val.as<sa_family_t>();
-                if (w == AF_INET)
-                    want |= WANT4;
-                else if(w == AF_INET6)
-                    want |= WANT6;
-            } catch (const std::exception& e) {};
-        }
-    } else {
-        want = -1;
-    }
-}
-
 void
 NetworkEngine::maintainRxBuffer(const TransId& tid)
 {
@@ -1689,44 +1361,6 @@ NetworkEngine::maintainRxBuffer(const TransId& tid)
     }
 }
 
-bool
-ParsedMessage::append(const ParsedMessage& block)
-{
-    bool ret(false);
-    for (const auto& ve : block.value_parts) {
-        auto part_val = value_parts.find(ve.first);
-        if (part_val == value_parts.end()
-            || part_val->second.second.size() >= part_val->second.first)
-            continue;
-        // TODO: handle out-of-order packets
-        if (ve.second.first != part_val->second.second.size()) {
-            //std::cout << "skipping out-of-order packet" << std::endl;
-            continue;
-        }
-        ret = true;
-        part_val->second.second.insert(part_val->second.second.end(),
-                                       ve.second.second.begin(),
-                                       ve.second.second.end());
-    }
-    return ret;
-}
-
-bool
-ParsedMessage::complete()
-{
-    for (auto& e : value_parts) {
-        //std::cout << "part " << e.first << ": " << e.second.second.size() << "/" << e.second.first << std::endl;
-        if (e.second.first > e.second.second.size())
-            return false;
-    }
-    for (auto& e : value_parts) {
-        msgpack::unpacked msg;
-        msgpack::unpack(msg, (const char*)e.second.second.data(), e.second.second.size());
-        values.emplace_back(std::make_shared<Value>(msg.get()));
-    }
-    return true;
-}
-
 
 } /* namespace net  */
 } /* namespace dht */

src/node.cpp

@@ -20,6 +20,7 @@
 
 #include "node.h"
 #include "request.h"
+#include "rng.h"
 
 #include <sstream>
 
@@ -29,6 +30,14 @@ constexpr std::chrono::minutes Node::NODE_EXPIRE_TIME;
 constexpr std::chrono::minutes Node::NODE_GOOD_TIME;
 constexpr std::chrono::seconds Node::MAX_RESPONSE_TIME;
 
+
+Node::Node(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& s, bool client)
+: id(id), addr(addr), sock(s), is_client(client), sockets_()
+{
+    crypto::random_device rd;
+    transaction_id = std::uniform_int_distribution<SocketId>{1}(rd);
+}
+
 /* This is our definition of a known-good node. */
 bool
 Node::isGood(time_point now) const
@@ -41,11 +50,9 @@ Node::isGood(time_point now) const
 bool
 Node::isPendingMessage() const
 {
-    for (auto w : requests_) {
-        if (auto r = w.lock()) {
-            if (r->pending())
-                return true;
-        }
+    for (auto r : requests_) {
+        if (r.second->pending())
+            return true;
     }
     return false;
 }
@@ -54,11 +61,9 @@ size_t
 Node::getPendingMessageCount() const
 {
     size_t count {0};
-    for (auto w : requests_) {
-        if (auto r = w.lock()) {
-            if (r->pending())
-                count++;
-        }
+    for (auto r : requests_) {
+        if (r.second->pending())
+            count++;
     }
     return count;
 }
@@ -73,27 +78,44 @@ Node::update(const SockAddr& new_addr, const Sp<TcpSocket>& s)
 
 /** To be called when a message was sent to the node */
 void
-Node::requested(std::shared_ptr<net::Request>& req)
+Node::requested(const Sp<net::Request>& req)
 {
-    requests_.emplace_back(req);
+    auto e = requests_.emplace(req->getTid(), req);
+    if (not e.second and req != e.first->second) {
+        // Should not happen !
+        // Try to handle this scenario as well as we can
+        e.first->second->setExpired();
+        e.first->second = req;
+    }
 }
 
 /** To be called when a message was received from the node.
  Req should be true if the message was an aswer to a request we made*/
 void
-Node::received(time_point now, std::shared_ptr<net::Request> req)
+Node::received(time_point now, const Sp<net::Request>& req)
 {
     time = now;
     expired_ = false;
     if (req) {
         reply_time = now;
-        for (auto it = requests_.begin(); it != requests_.end();) {
-            auto r = it->lock();
-            if (not r or r == req)
-                it = requests_.erase(it);
-            else
-                ++it;
-        }
+        requests_.erase(req->getTid());
+    }
+}
+
+Sp<net::Request>
+Node::getRequest(const net::TransId& tid)
+{
+    auto it = requests_.find(tid);
+    return it != requests_.end() ? it->second : nullptr;
+}
+
+void
+Node::cancelRequest(const Sp<net::Request>& req)
+{
+    if (req) {
+        req->cancel();
+        closeSocket(req->closeSocket());
+        requests_.erase(req->getTid());
     }
 }
 
@@ -101,13 +123,41 @@ void
 Node::setExpired()
 {
     expired_ = true;
-    for (auto w : requests_) {
-        if (auto r = w.lock())
-            r->setExpired();
+    for (auto r : requests_) {
+        r.second->setExpired();
     }
     requests_.clear();
+    sockets_.clear();
 }
 
+SocketId
+Node::openSocket(SocketCb&& cb)
+{
+    if (++transaction_id == 0)
+        transaction_id = 1;
+
+    auto sock = Socket(std::move(cb));
+    auto s = sockets_.emplace(transaction_id, std::move(sock));
+    if (not s.second)
+        s.first->second = std::move(sock);
+    return transaction_id;
+}
+
+Socket*
+Node::getSocket(SocketId id)
+{
+    auto it = sockets_.find(id);
+    return it == sockets_.end() ? nullptr : &it->second;
+}
+
+void
+Node::closeSocket(SocketId id)
+{
+    if (id) {
+        sockets_.erase(id);
+        //DHT_LOG.w("Closing socket (tid: %d), %lu remaining", socket->id, sockets_.size());
+    }
+}
 
 std::string
 Node::toString() const

src/node_cache.cpp

@@ -20,44 +20,47 @@
 
 namespace dht {
 
-std::shared_ptr<Node>
+Sp<Node>
 NodeCache::getNode(const InfoHash& id, sa_family_t family) {
-    return (family == AF_INET ? cache_4 : cache_6).getNode(id);
+    return cache(family).getNode(id);
 }
 
-std::shared_ptr<Node>
-NodeCache::getNode(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& sock, time_point now, bool confirm) {
+Sp<Node>
+NodeCache::getNode(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& sock, time_point now, bool confirm, bool client) {
     if (id == zeroes)
-        return std::make_shared<Node>(id, addr, sock);
-    return (addr.getFamily() == AF_INET ? cache_4 : cache_6).getNode(id, addr, sock, now, confirm);
+        return std::make_shared<Node>(id, addr);
+    return cache(addr.getFamily()).getNode(id, addr, sock, now, confirm, client);
 }
 
-std::vector<std::shared_ptr<Node>>
-NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) {
-    const auto& c = (sa_f == AF_INET ? cache_4 : cache_6);
+std::vector<Sp<Node>>
+NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const
+{
+    const auto& c = cache(sa_f);
 
-    std::vector<std::shared_ptr<Node>> nodes;
+    std::vector<Sp<Node>> nodes;
     nodes.reserve(std::min(c.size(), count));
     NodeMap::const_iterator it;
+    auto dec_it = [&c](NodeMap::const_iterator& it) {
+        auto ret = it;
+        it = (it == c.cbegin()) ? c.cend() : std::prev(it);
+        return ret;
+    };
 
     auto it_p = c.lower_bound(id),
-         it_n = it_p;
-    if (it_p != c.begin()) /* Create 2 separate iterator if we could */
-        --it_p;
+        it_n = it_p;
+    if (not c.empty())
+        dec_it(it_p); /* Create 2 separate iterator if we could */
 
-    while (nodes.size() < count and (it_n != c.end() or it_p != c.end())) {
+    while (nodes.size() < count and (it_n != c.cend() or it_p != c.cend())) {
         /* If one of the iterator is at the end, then take the other one
            If they are both in middle of somewhere comapre both and take
            the closest to the id. */
-        if (it_p == c.end())       it = it_n++;
-        else if (it_n == c.end())  it = it_p--;
-        else                       it = id.xorCmp(it_p->first, it_n->first) < 0 ? it_p-- : it_n++;
-
-        if (it == c.begin())
-            it_p = c.end();
+        if (it_p == c.cend())       it = it_n++;
+        else if (it_n == c.cend())  it = dec_it(it_p);
+        else                        it = id.xorCmp(it_p->first, it_n->first) < 0 ? dec_it(it_p) : it_n++;
 
         if (auto n = it->second.lock())
-            if ( not n->isExpired() )
+            if ( not n->isExpired() and not n->isClient() )
                 nodes.emplace_back(std::move(n));
     }
 
@@ -71,7 +74,7 @@ NodeCache::clearBadNodes(sa_family_t family)
         clearBadNodes(AF_INET);
         clearBadNodes(AF_INET6);
     } else {
-        (family == AF_INET ? cache_4 : cache_6).clearBadNodes();
+        cache(family).clearBadNodes();
     }
 }
 
@@ -86,7 +89,7 @@ NodeCache::closeAll(sa_family_t family)
     }
 }
 
-std::shared_ptr<Node>
+Sp<Node>
 NodeCache::NodeMap::getNode(const InfoHash& id)
 {
     auto wn = find(id);
@@ -98,15 +101,15 @@ NodeCache::NodeMap::getNode(const InfoHash& id)
     return {};
 }
 
-std::shared_ptr<Node>
-NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& sock, time_point now, bool confirm)
+Sp<Node>
+NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& sock, time_point now, bool confirm, bool client)
 {
     auto it = emplace(id, std::weak_ptr<Node>{});
     auto node = it.first->second.lock();
     if (not node) {
-        node = std::make_shared<Node>(id, addr, sock);
+        node = std::make_shared<Node>(id, addr, sock, client);
         it.first->second = node;
-    } else if (confirm || node->time < now - Node::NODE_EXPIRE_TIME) {
+    } else if (confirm or node->isOld(now)) {
         node->update(addr, sock);
     }
     return node;

src/parsed_message.h

@@ -0,0 +1,309 @@
+/*
+ *  Copyright (C) 2017 Savoir-faire Linux Inc.
+ *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+ #pragma once
+
+#include "infohash.h"
+#include "sockaddr.h"
+#include "net.h"
+
+#include <map>
+
+namespace dht {
+namespace net {
+
+enum class MessageType {
+    Error = 0,
+    Reply,
+    Ping,
+    FindNode,
+    GetValues,
+    AnnounceValue,
+    Refresh,
+    Listen,
+    ValueData
+};
+
+struct ParsedMessage {
+    MessageType type;
+    /* Node ID of the sender */
+    InfoHash id;
+    /* Network id */
+    NetId network {0};
+    /** Is a client node */
+    bool is_client {false};
+    /* hash for which values are requested */
+    InfoHash info_hash;
+    /* target id around which to find nodes */
+    InfoHash target;
+    /* transaction id */
+    TransId tid;
+    /* tid for packets going through request socket */
+    TransId socket_id;
+    /* security token */
+    Blob token;
+    /* the value id (announce confirmation) */
+    Value::Id value_id;
+    /* time when value was first created */
+    time_point created { time_point::max() };
+    /* IPv4 nodes in response to a 'find' request */
+    Blob nodes4_raw, nodes6_raw;
+    std::vector<Sp<Node>> nodes4, nodes6;
+    /* values to store or retreive request */
+    std::vector<Sp<Value>> values;
+    /* index for fields values */
+    std::vector<Sp<FieldValueIndex>> fields;
+    /** When part of the message header: {index -> (total size, {})}
+     *  When part of partial value data: {index -> (offset, part_data)} */
+    std::map<unsigned, std::pair<unsigned, Blob>> value_parts;
+    /* query describing a filter to apply on values. */
+    Query query;
+    /* states if ipv4 or ipv6 request */
+    want_t want;
+    /* error code in case of error */
+    uint16_t error_code;
+    /* reported address by the distant node */
+    std::string ua;
+    SockAddr addr;
+    void msgpack_unpack(msgpack::object o);
+
+    bool append(const ParsedMessage& block);
+    bool complete();
+};
+
+bool
+ParsedMessage::append(const ParsedMessage& block)
+{
+    bool ret(false);
+    for (const auto& ve : block.value_parts) {
+        auto part_val = value_parts.find(ve.first);
+        if (part_val == value_parts.end()
+            || part_val->second.second.size() >= part_val->second.first)
+            continue;
+        // TODO: handle out-of-order packets
+        if (ve.second.first != part_val->second.second.size()) {
+            //std::cout << "skipping out-of-order packet" << std::endl;
+            continue;
+        }
+        ret = true;
+        part_val->second.second.insert(part_val->second.second.end(),
+                                       ve.second.second.begin(),
+                                       ve.second.second.end());
+    }
+    return ret;
+}
+
+bool
+ParsedMessage::complete()
+{
+    for (auto& e : value_parts) {
+        //std::cout << "part " << e.first << ": " << e.second.second.size() << "/" << e.second.first << std::endl;
+        if (e.second.first > e.second.second.size())
+            return false;
+    }
+    for (auto& e : value_parts) {
+        msgpack::unpacked msg;
+        msgpack::unpack(msg, (const char*)e.second.second.data(), e.second.second.size());
+        values.emplace_back(std::make_shared<Value>(msg.get()));
+    }
+    return true;
+}
+
+void
+ParsedMessage::msgpack_unpack(msgpack::object msg)
+{
+    auto y = findMapValue(msg, "y");
+    auto r = findMapValue(msg, "r");
+    auto e = findMapValue(msg, "e");
+    auto v = findMapValue(msg, "p");
+
+    if (auto t = findMapValue(msg, "t"))
+        tid = unpackTid(*t);
+
+    if (auto rv = findMapValue(msg, "v"))
+        ua = rv->as<std::string>();
+
+    if (auto netid = findMapValue(msg, "n"))
+        network = netid->as<NetId>();
+
+    if (auto is_client_v = findMapValue(msg, "s"))
+        is_client = is_client_v->as<bool>();
+
+    std::string q;
+    if (auto rq = findMapValue(msg, "q")) {
+        if (rq->type != msgpack::type::STR)
+            throw msgpack::type_error();
+        q = rq->as<std::string>();
+    }
+
+    if (e)
+        type = MessageType::Error;
+    else if (r)
+        type = MessageType::Reply;
+    else if (v)
+        type = MessageType::ValueData;
+    else if (y and y->as<std::string>() != "q")
+        throw msgpack::type_error();
+    else if (q == "ping")
+        type = MessageType::Ping;
+    else if (q == "find")
+        type = MessageType::FindNode;
+    else if (q == "get")
+        type = MessageType::GetValues;
+    else if (q == "listen")
+        type = MessageType::Listen;
+    else if (q == "put")
+        type = MessageType::AnnounceValue;
+    else if (q == "refresh")
+        type = MessageType::Refresh;
+    else
+        throw msgpack::type_error();
+
+    if (type == MessageType::ValueData) {
+        if (v->type != msgpack::type::MAP)
+            throw msgpack::type_error();
+        for (size_t i = 0; i < v->via.map.size; ++i) {
+            auto& vdat = v->via.map.ptr[i];
+            auto o = findMapValue(vdat.val, "o");
+            auto d = findMapValue(vdat.val, "d");
+            if (not o or not d)
+                continue;
+            value_parts.emplace(vdat.key.as<unsigned>(), std::pair<size_t, Blob>(o->as<size_t>(), unpackBlob(*d)));
+        }
+        return;
+    }
+
+    auto a = findMapValue(msg, "a");
+    if (!a && !r && !e)
+        throw msgpack::type_error();
+    auto& req = a ? *a : (r ? *r : *e);
+
+    if (e) {
+        if (e->type != msgpack::type::ARRAY)
+            throw msgpack::type_error();
+        error_code = e->via.array.ptr[0].as<uint16_t>();
+    }
+
+    if (auto t = findMapValue(req, "sid"))
+        socket_id = unpackTid(*t);
+
+    if (auto rid = findMapValue(req, "id"))
+        id = {*rid};
+
+    if (auto rh = findMapValue(req, "h"))
+        info_hash = {*rh};
+
+    if (auto rtarget = findMapValue(req, "target"))
+        target = {*rtarget};
+
+    if (auto rquery = findMapValue(req, "q"))
+        query.msgpack_unpack(*rquery);
+
+    if (auto otoken = findMapValue(req, "token"))
+        token = unpackBlob(*otoken);
+
+    if (auto vid = findMapValue(req, "vid"))
+        value_id = vid->as<Value::Id>();
+
+    if (auto rnodes4 = findMapValue(req, "n4"))
+        nodes4_raw = unpackBlob(*rnodes4);
+
+    if (auto rnodes6 = findMapValue(req, "n6"))
+        nodes6_raw = unpackBlob(*rnodes6);
+
+    if (auto sa = findMapValue(req, "sa")) {
+        if (sa->type != msgpack::type::BIN)
+            throw msgpack::type_error();
+        auto l = sa->via.bin.size;
+        if (l == sizeof(in_addr)) {
+            addr.setFamily(AF_INET);
+            auto& a = addr.getIPv4();
+            std::fill_n((uint8_t*)&a, sizeof(a), 0);
+            a.sin_port = 0;
+            std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin_addr);
+        } else if (l == sizeof(in6_addr)) {
+            addr.setFamily(AF_INET6);
+            auto& a = addr.getIPv6();
+            std::fill_n((uint8_t*)&a, sizeof(a), 0);
+            a.sin6_port = 0;
+            std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin6_addr);
+        }
+    } else
+        addr = {};
+
+    if (auto rcreated = findMapValue(req, "c"))
+        created = from_time_t(rcreated->as<std::time_t>());
+
+    if (auto rvalues = findMapValue(req, "values")) {
+        if (rvalues->type != msgpack::type::ARRAY)
+            throw msgpack::type_error();
+        for (size_t i = 0; i < rvalues->via.array.size; i++) {
+            auto& packed_v = rvalues->via.array.ptr[i];
+            if (packed_v.type == msgpack::type::POSITIVE_INTEGER) {
+                // Skip oversize values with a small margin for header overhead
+                if (packed_v.via.u64 > MAX_VALUE_SIZE + 32)
+                    continue;
+                value_parts.emplace(i, std::make_pair(packed_v.via.u64, Blob{}));
+            } else {
+                try {
+                    values.emplace_back(std::make_shared<Value>(rvalues->via.array.ptr[i]));
+                } catch (const std::exception& e) {
+                     //DHT_LOG_WARN("Error reading value: %s", e.what());
+                }
+            }
+        }
+    } else if (auto raw_fields = findMapValue(req, "fields")) {
+        if (auto rfields = findMapValue(*raw_fields, "f")) {
+            auto vfields = rfields->as<std::set<Value::Field>>();
+            if (auto rvalues = findMapValue(*raw_fields, "v")) {
+                if (rvalues->type != msgpack::type::ARRAY)
+                    throw msgpack::type_error();
+                size_t val_num = rvalues->via.array.size / vfields.size();
+                for (size_t i = 0; i < val_num; ++i) {
+                    try {
+                        auto v = std::make_shared<FieldValueIndex>();
+                        v->msgpack_unpack_fields(vfields, *rvalues, i*vfields.size());
+                        fields.emplace_back(std::move(v));
+                    } catch (const std::exception& e) { }
+                }
+            }
+        } else {
+            throw msgpack::type_error();
+        }
+    }
+
+    if (auto w = findMapValue(req, "w")) {
+        if (w->type != msgpack::type::ARRAY)
+            throw msgpack::type_error();
+        want = 0;
+        for (unsigned i=0; i<w->via.array.size; i++) {
+            auto& val = w->via.array.ptr[i];
+            try {
+                auto w = val.as<sa_family_t>();
+                if (w == AF_INET)
+                    want |= WANT4;
+                else if(w == AF_INET6)
+                    want |= WANT6;
+            } catch (const std::exception& e) {};
+        }
+    } else {
+        want = -1;
+    }
+}
+
+} /* namespace net  */
+} /* namespace dht */

src/request.h

@@ -20,34 +20,15 @@
 #pragma once
 
 #include "net.h"
+#include "value.h"
 
 namespace dht {
+struct Node;
 namespace net {
 
 class NetworkEngine;
 struct ParsedMessage;
 
-/*!
- * @class   Socket
- * @brief   Open route to a node for continous incoming packets.
- * @details
- * A socket lets a remote node send us continuous packets treated using a
- * given callback. This is intended to provide an easy management of
- * specific updates nodes can send. For e.g, this is used in the case of the
- * "listen" operation for treating updates a node has for a given storage.
- */
-struct Socket {
-    Socket() {}
-    Socket(std::shared_ptr<Node> node,
-           TransId id,
-           std::function<void(const std::shared_ptr<Node>&, ParsedMessage&&)> on_receive) :
-        node(node), id(id), on_receive(on_receive) {}
-
-    std::shared_ptr<Node> node;
-    TransId id;
-    std::function<void(const std::shared_ptr<Node>&, ParsedMessage&&)> on_receive {};
-};
-
 /*!
  * @class   Request
  * @brief   An atomic request destined to a node.
@@ -59,7 +40,7 @@ struct Socket {
 struct Request {
     friend class dht::net::NetworkEngine;
 
-    std::shared_ptr<Node> node {};             /* the node to whom the request is destined. */
+    Sp<Node> node {};             /* the node to whom the request is destined. */
     time_point reply_time {time_point::min()}; /* time when we received the response to the request. */
 
     enum class State
@@ -78,14 +59,19 @@ struct Request {
     State getState() const { return state_; }
 
     Request(State state = State::PENDING) : state_(state) {}
-    Request(TransId tid,
-            std::shared_ptr<Node> node,
+    Request(const TransId& tid,
+            Sp<Node> node,
             Blob&& msg,
             std::function<void(const Request&, ParsedMessage&&)> on_done,
             std::function<void(const Request&, bool)> on_expired,
-            std::shared_ptr<Socket> socket = {}) :
+            uint32_t socket = 0) :
         node(node), on_done(on_done), on_expired(on_expired), tid(tid), msg(std::move(msg)), socket(socket) { }
 
+    TransId getTid() const { return tid; }
+
+    uint32_t getSocket() { return socket; }
+    uint32_t closeSocket() { auto ret = socket; socket = 0; return ret; }
+
     void setExpired() {
         if (pending()) {
             state_ = Request::State::EXPIRED;
@@ -132,7 +118,7 @@ private:
 
     const TransId tid; /* the request id. */
     Blob msg {};                      /* the serialized message. */
-    std::shared_ptr<Socket> socket;   /* the socket used for further reponses. */
+    uint32_t socket;   /* the socket used for further reponses. */
 };
 
 } /* namespace net  */

src/routing_table.cpp

@@ -1,4 +1,24 @@
+/*
+ *  Copyright (C) 2014-2017 Savoir-faire Linux Inc.
+ *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
 #include "routing_table.h"
+
+#include "network_engine.h"
 #include "rng.h"
 
 #include <memory>
@@ -35,6 +55,15 @@ Bucket::randomNode()
     return nodes.back();
 }
 
+void Bucket::sendCachedPing(net::NetworkEngine& ne)
+{
+    if (not cached)
+        return;
+    //DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", cached->toString().c_str());
+    ne.sendPing(cached, nullptr, nullptr);
+    cached = {};
+}
+
 InfoHash
 RoutingTable::randomId(const RoutingTable::const_iterator& it) const
 {
@@ -172,4 +201,66 @@ RoutingTable::split(const RoutingTable::iterator& b)
     return true;
 }
 
+bool
+RoutingTable::onNewNode(const Sp<Node>& node, int confirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne) {
+    auto b = findBucket(node->id);
+    if (b == end()) return false;
+
+    if (confirm == 2)
+        b->time = now;
+
+    for (auto& n : b->nodes) {
+        if (n == node)
+            return false;
+    }
+
+    bool mybucket = contains(b, myid);
+    if (mybucket) {
+        grow_time = now;
+        //scheduler.edit(nextNodesConfirmation, now);
+    }
+
+    if (b->nodes.size() >= TARGET_NODES) {
+        /* Try to get rid of an expired node. */
+        for (auto& n : b->nodes)
+            if (n->isExpired()) {
+                n = node;
+                return true;
+            }
+        /* Bucket full.  Ping a dubious node */
+        bool dubious = false;
+        for (auto& n : b->nodes) {
+            /* Pick the first dubious node that we haven't pinged in the
+               last 9 seconds.  This gives nodes the time to reply, but
+               tends to concentrate on the same nodes, so that we get rid
+               of bad nodes fast. */
+            if (not n->isGood(now)) {
+                dubious = true;
+                if (not n->isPendingMessage()) {
+                    //DHT_LOG.d(n->id, "[node %s] sending ping to dubious node", n->toString().c_str());
+                    ne.sendPing(n, nullptr, nullptr);
+                    break;
+                }
+            }
+        }
+
+        if ((mybucket || (is_client and depth(b) < 6)) && (!dubious || size() == 1)) {
+            //DHT_LOG.d("Splitting from depth %u", depth(b));
+            b->sendCachedPing(ne);
+            split(b);
+            return onNewNode(node, confirm, now, myid, ne);
+        }
+
+        /* No space for this node.  Cache it away for later. */
+        if (confirm or not b->cached)
+            b->cached = node;
+    } else {
+        /* Create a new node. */
+        b->nodes.emplace_front(node);
+    }
+    return true;
+}
+
+
+
 }

src/search.h

@@ -76,7 +76,19 @@ struct Dht::SearchNode {
                                                       node is a new candidate for inclusion. */
 
     SearchNode() : node() {}
+    SearchNode(const SearchNode&) = delete;
+    SearchNode(SearchNode&&) = default;
+    SearchNode& operator=(const SearchNode&) = delete;
+    SearchNode& operator=(SearchNode&&) = default;
+
     SearchNode(const Sp<Node>& node) : node(node) {}
+    ~SearchNode() {
+        if (node) {
+            cancelGet();
+            cancelListen();
+            cancelAnnounce();
+        }
+    }
 
     /**
      * Can we use this node to listen/announce now ?
@@ -173,6 +185,15 @@ struct Dht::SearchNode {
         }
     }
 
+    void cancelGet() {
+        for (const auto& status : getStatus) {
+            if (status.second->pending()) {
+                node->cancelRequest(status.second);
+            }
+        }
+        getStatus.clear();
+    }
+
     /**
      * Tells if a request in the status map is expired.
      *
@@ -180,12 +201,12 @@ struct Dht::SearchNode {
      *
      * @return true if there exists an expired request, else false.
      */
-    bool expired(const SyncStatus& status) const {
+    /*static bool expired(const SyncStatus& status) const {
         return std::find_if(status.begin(), status.end(),
             [](const SyncStatus::value_type& r){
                 return r.second and r.second->expired();
             }) != status.end();
-    }
+    }*/
 
     /**
      * Tells if a request in the status map is pending.
@@ -194,19 +215,30 @@ struct Dht::SearchNode {
      *
      * @return true if there exists an expired request, else false.
      */
-    bool pending(const SyncStatus& status) const {
+    static bool pending(const SyncStatus& status) {
         return std::find_if(status.begin(), status.end(),
             [](const SyncStatus::value_type& r){
                 return r.second and r.second->pending();
             }) != status.end();
     }
 
+    bool pendingGet() const { return pending(getStatus); }
+
     bool isAnnounced(Value::Id vid) const {
         auto ack = acked.find(vid);
         if (ack == acked.end() or not ack->second.first)
             return false;
         return ack->second.first->completed();
     }
+    void cancelAnnounce() {
+        for (const auto& status : acked) {
+            const auto& req = status.second.first;
+            if (req and req->pending()) {
+                node->cancelRequest(req);
+            }
+        }
+        acked.clear();
+    }
 
     bool isListening(time_point now) const {
         auto ls = listenStatus.begin();
@@ -229,6 +261,18 @@ struct Dht::SearchNode {
             return false;
         return listen_status->second->reply_time + LISTEN_EXPIRE_TIME > now;
     }
+    void cancelListen() {
+        for (const auto& status : listenStatus)
+            node->cancelRequest(status.second);
+        listenStatus.clear();
+    }
+    void cancelListen(const Sp<Query>& query) {
+        auto it = listenStatus.find(query);
+        if (it != listenStatus.end()) {
+            node->cancelRequest(it->second);
+            listenStatus.erase(it);
+        }
+    }
 
     /**
      * Assumng the node is synced, should a "put" request be sent to this node now ?
@@ -319,7 +363,7 @@ struct Dht::Search {
     unsigned currentlySolicitedNodeCount() const {
         unsigned count = 0;
         for (const auto& n : nodes)
-            if (not n.isBad() and n.pending(n.getStatus))
+            if (not n.isBad() and n.pendingGet())
                 count++;
         return count;
     }
@@ -382,6 +426,21 @@ struct Dht::Search {
     bool isAnnounced(Value::Id id) const;
     bool isListening(time_point now) const;
 
+    void cancelListen(size_t token) {
+        Sp<Query> query;
+        const auto& ll = listeners.find(token);
+        if (ll != listeners.cend()) {
+            query = ll->second.query;
+            listeners.erase(ll);
+        }
+        for (auto& sn : nodes) {
+            if (listeners.empty())
+                sn.cancelListen();
+            else if (query)
+                sn.cancelListen(query);
+        }
+    }
+
     /**
      * @return The number of non-good search nodes.
      */
@@ -430,10 +489,10 @@ struct Dht::Search {
      *
      * @return true if a node has been removed, else false.
      */
-    bool removeExpiredNode(time_point now) {
+    bool removeExpiredNode(const time_point& now) {
         for (auto e = nodes.cend(); e != nodes.cbegin();) {
             const Node& n = *(--e)->node;
-            if (n.isExpired() and n.time + Node::NODE_EXPIRE_TIME < now) {
+            if (n.isRemovable(now)) {
                 //std::cout << "Removing expired node " << n.id << " from IPv" << (af==AF_INET?'4':'6') << " search " << id << std::endl;
                 nodes.erase(e);
                 return true;
@@ -585,7 +644,7 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
             step_time = time_point::min();
         }
         n = nodes.insert(n, SearchNode(snode));
-        node.time = now;
+        node.setTime(now);
         new_search_node = true;
         if (node.isExpired()) {
             if (not expired)
@@ -672,7 +731,7 @@ Dht::Search::getUpdateTime(time_point now) const
     for (const auto& sn : nodes) {
         if (sn.node->isExpired() or (sn.candidate and t >= TARGET_NODES))
             continue;
-        auto pending = sn.pending(sn.getStatus);
+        auto pending = sn.pendingGet();
         if (sn.last_get_reply < std::max(now - Node::NODE_EXPIRE_TIME, last_get) or pending) {
             // not isSynced
             if (not pending and solicited_nodes < MAX_REQUESTED_SEARCH_NODES)

src/securedht.cpp

@@ -76,7 +76,7 @@ SecureDht::~SecureDht()
 ValueType
 SecureDht::secureType(ValueType&& type)
 {
-    type.storePolicy = [this,type](InfoHash id, Sp<Value>& v, InfoHash nid, const sockaddr* a, socklen_t al) {
+    type.storePolicy = [this,type](InfoHash id, Sp<Value>& v, const InfoHash& nid, const SockAddr& a) {
         if (v->isSigned()) {
             if (!v->owner or !v->owner->checkSignature(v->getToSign(), v->signature)) {
                 DHT_LOG.WARN("Signature verification failed");
@@ -85,11 +85,11 @@ SecureDht::secureType(ValueType&& type)
             else
                 DHT_LOG.WARN("Signature verification succeeded");
         }
-        return type.storePolicy(id, v, nid, a, al);
+        return type.storePolicy(id, v, nid, a);
     };
-    type.editPolicy = [this,type](InfoHash id, const Sp<Value>& o, Sp<Value>& n, InfoHash nid, const sockaddr* a, socklen_t al) {
+    type.editPolicy = [this,type](InfoHash id, const Sp<Value>& o, Sp<Value>& n, const InfoHash& nid, const SockAddr& a) {
         if (!o->isSigned())
-            return type.editPolicy(id, o, n, nid, a, al);
+            return type.editPolicy(id, o, n, nid, a);
         if (o->owner != n->owner) {
             DHT_LOG.WARN("Edition forbidden: owner changed.");
             return false;

src/utils.cpp

@@ -19,6 +19,7 @@
 #include "utils.h"
 #include "sockaddr.h"
 #include "default_types.h"
+#include "net.h"
 #include "uv_utils.h"
 
 /* An IPv4 equivalent to IN6_IS_ADDR_UNSPECIFIED */
@@ -41,7 +42,7 @@ print_addr(const sockaddr* sa, socklen_t slen)
             out << "[" << hbuf << "]";
         else
             out << hbuf;
-        if (strcmp(sbuf, "0"))
+        if (std::strcmp(sbuf, "0"))
             out << ":" << sbuf;
     } else
         out << "[invalid address]";
@@ -54,11 +55,6 @@ print_addr(const sockaddr_storage& ss, socklen_t sslen)
     return print_addr((const sockaddr*)&ss, sslen);
 }
 
-std::string
-printAddr(const SockAddr& addr) {
-    return print_addr((const sockaddr*)&addr.first, addr.second);
-}
-
 bool
 SockAddr::isUnspecified() const
 {
@@ -219,4 +215,18 @@ findMapValue(msgpack::object& map, const std::string& key) {
     return nullptr;
 }
 
+namespace net {
+
+TransId
+unpackTid(msgpack::object& o) {
+    switch (o.type) {
+    case msgpack::type::POSITIVE_INTEGER:
+        return o.as<uint32_t>();
+    default:
+        return o.as<std::array<char, 4>>();
+    }
+}
+
+}
+
 }

src/value.cpp

@@ -35,6 +35,7 @@ Value::Filter bindFilterRaw(FilterRaw raw_filter, void* user_data) {
 
 std::ostream& operator<< (std::ostream& s, const Value& v)
 {
+    auto flags(s.flags());
     s << "Value[id:" << std::hex << v.id << std::dec << " ";
     if (v.isEncrypted())
         s << "encrypted ";
@@ -63,13 +64,14 @@ std::ostream& operator<< (std::ostream& s, const Value& v)
         }
     }
     s << "]";
+    s.flags(flags);
     return s;
 }
 
 const ValueType ValueType::USER_DATA = {0, "User Data"};
 
 bool
-ValueType::DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr<Value>& v, InfoHash, const sockaddr*, socklen_t)
+ValueType::DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr<Value>& v, const InfoHash&, const SockAddr&)
 {
     return v->size() <= MAX_VALUE_SIZE;
 }
@@ -257,9 +259,12 @@ std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi) {
     os << "Index[";
     for (auto v = fvi.index.begin(); v != fvi.index.end(); ++v) {
         switch (v->first) {
-            case Value::Field::Id:
+            case Value::Field::Id: {
+                auto flags(os.flags());
                 os << "Id:" << std::hex << v->second.getInt();
+                os.flags(flags);
                 break;
+            }
             case Value::Field::ValueType:
                 os << "ValueType:" << v->second.getInt();
                 break;