uv

CMakeLists.txt

@@ -24,6 +24,8 @@ option (OPENDHT_DOCUMENTATION "Create and install the HTML based API documentati
 list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 find_package (Threads)
 find_package (PkgConfig)
+pkg_search_module(libuv libuv)
+pkg_search_module(libsodium libsodium)
 find_package (GnuTLS 3.3 REQUIRED)
 pkg_search_module (Nettle nettle)
 find_package (Msgpack 1.2 REQUIRED)
@@ -31,7 +33,6 @@ if (OPENDHT_TOOLS)
     find_package (Readline 6 REQUIRED)
 endif ()
 if (NOT OPENDHT_ARGON2)
-    find_package(PkgConfig)
     pkg_search_module(argon2 libargon2)
     if (argon2_FOUND)
         message("Argon2 found " ${argon2_req})
@@ -43,7 +44,7 @@ if (NOT OPENDHT_ARGON2)
 endif ()
 
 # Build flags
-set (CMAKE_CXX_STANDARD 11)
+set (CMAKE_CXX_STANDARD 14)
 set (CMAKE_CXX_STANDARD_REQUIRED on)
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden")
 if (OPENDHT_SANITIZE)
@@ -77,7 +78,13 @@ endif ()
 if (Nettle_INCLUDE_DIRS)
     include_directories (SYSTEM "${Nettle_INCLUDE_DIRS}")
 endif ()
-link_directories (${Nettle_LIBRARY_DIRS})
+if (libuv_INCLUDE_DIRS)
+    include_directories (SYSTEM "${libuv_INCLUDE_DIRS}")
+endif ()
+if (libsodium_INCLUDE_DIRS)
+    include_directories (SYSTEM "${libsodium_INCLUDE_DIRS}")
+endif ()
+link_directories (${libuv_LIBRARY_DIRS} ${libsodium_LIBRARY_DIRS} ${Nettle_LIBRARY_DIRS})
 include_directories (
     ./
     include/
@@ -177,7 +184,7 @@ if (OPENDHT_STATIC)
         target_link_libraries(opendht-static ${argon2_LIBRARIES})
         target_include_directories(opendht-static SYSTEM PRIVATE ${argon2_INCLUDE_DIRS})
     endif ()
-    target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES})
+    target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${libuv_LIBRARIES} ${libsodium_LIBRARIES})
     install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht)
 endif ()
 
@@ -198,7 +205,7 @@ if (OPENDHT_SHARED)
         target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES})
         target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS})
     endif ()
-    target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES})
+    target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${libuv_LIBRARIES} ${libsodium_LIBRARIES})
     install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht)
 endif ()
 

include/opendht/callbacks.h

@@ -49,6 +49,12 @@ struct OPENDHT_PUBLIC NodeStats {
     std::string toString() const;
 };
 
+
+struct OPENDHT_PUBLIC NetworkConfig {
+    std::string bind_addr {"::0"};
+    in_port_t bind_port {0};
+};
+
 /**
  * Dht configuration.
  */
@@ -68,6 +74,8 @@ struct OPENDHT_PUBLIC Config {
 
     /** Makes the DHT responsible to maintain its stored values. Consumes more ressources. */
     bool maintain_storage;
+
+    NetworkConfig network_config;
 };
 
 /**
@@ -87,6 +95,7 @@ using QueryCallback = std::function<bool(const std::vector<std::shared_ptr<Field
 using GetCallback = std::function<bool(const std::vector<std::shared_ptr<Value>>& values)>;
 using GetCallbackSimple = std::function<bool(std::shared_ptr<Value> value)>;
 using ShutdownCallback = std::function<void()>;
+using StatusCallback = std::function<void(NodeStatus, NodeStatus)>;
 
 using CertificateStoreQuery = std::function<std::vector<std::shared_ptr<crypto::Certificate>>(const InfoHash& pk_id)>;
 
@@ -95,7 +104,7 @@ typedef bool (*GetCallbackRaw)(std::shared_ptr<Value>, void *user_data);
 OPENDHT_PUBLIC GetCallbackSimple bindGetCb(GetCallbackRaw raw_cb, void* user_data);
 OPENDHT_PUBLIC GetCallback bindGetCb(GetCallbackSimple cb);
 
-using DoneCallback = std::function<void(bool success, const std::vector<std::shared_ptr<Node>>& nodes)>;
+using DoneCallback = std::function<void(bool success, std::vector<std::shared_ptr<Node>>&& nodes)>;
 typedef void (*DoneCallbackRaw)(bool, std::vector<std::shared_ptr<Node>>*, void *user_data);
 typedef void (*ShutdownCallbackRaw)(void *user_data);
 typedef void (*DoneCallbackSimpleRaw)(bool, void *user_data);

include/opendht/crypto.h

@@ -28,6 +28,8 @@ extern "C" {
 #include <gnutls/x509.h>
 }
 
+#include <sodium.h>
+
 #include <vector>
 #include <memory>
 
@@ -35,6 +37,8 @@ extern "C" {
 #include <iso646.h>
 #endif
 
+#include <sys/mman.h>
+
 namespace dht {
 
 /**
@@ -500,11 +504,19 @@ public:
     secure_vector() {}
     secure_vector(secure_vector<T> const&) = default;
     secure_vector(secure_vector<T> &&) = default;
-    explicit secure_vector(unsigned size): data_(size) {}
-    explicit secure_vector(unsigned size, T _item): data_(size, _item) {}
-    explicit secure_vector(const std::vector<T>& c): data_(c) {}
-    secure_vector(std::vector<T>&& c): data_(std::move(c)) {}
-    ~secure_vector() { clean(); }
+    explicit secure_vector(unsigned size): data_(size) {
+        mlock(data_.data(), data_.size()*sizeof(T));
+    }
+    explicit secure_vector(unsigned size, T _item): data_(size, _item) {
+        mlock(data_.data(), data_.size()*sizeof(T));
+    }
+    explicit secure_vector(const std::vector<T>& c): data_(c) {
+        mlock(data_.data(), data_.size()*sizeof(T));
+    }
+    secure_vector(std::vector<T>&& c): data_(std::move(c)) {
+        mlock(data_.data(), data_.size()*sizeof(T));
+    }
+    ~secure_vector() { clean(); munlock(data_.data(), data_.size()*sizeof(T)); }
 
     static secure_vector<T> getRandom(size_t size) {
         secure_vector<T> ret(size/sizeof(T));
@@ -517,24 +529,30 @@ public:
         if (&c == this)
             return *this;
         clean();
+        munlock(data_.data(), data_.size()*sizeof(T));
         data_ = c.data_;
+        mlock(data_.data(), data_.size()*sizeof(T));
         return *this;
     }
     secure_vector<T>& operator=(secure_vector<T>&& c) {
         if (&c == this)
             return *this;
         clean();
+        munlock(data_.data(), data_.size()*sizeof(T));
         data_ = std::move(c.data_);
         return *this;
     }
     secure_vector<T>& operator=(std::vector<T>&& c) {
         clean();
+        munlock(data_.data(), data_.size()*sizeof(T));
         data_ = std::move(c);
+        mlock(data_.data(), data_.size()*sizeof(T));
         return *this;
     }
     std::vector<T>& writable() { clean(); return data_; }
     const std::vector<T>& makeInsecure() const { return data_; }
     const uint8_t* data() const { return data_.data(); }
+    uint8_t* data() { return data_.data(); }
 
     void clean() {
         clean(data_.begin(), data_.end());
@@ -551,14 +569,17 @@ public:
         if (s < data_.size()) {
             //shrink
             clean(data_.begin()+s, data_.end());
+            munlock(data_.data()+s, (data_.size()-s)*sizeof(T));
             data_.resize(s);
         } else {
             //grow
             auto data = std::move(data_); // move protected data
             clear();
             data_.resize(s);
+            mlock(data_.data(), data_.size()*sizeof(T));
             std::copy(data.begin(), data.end(), data_.begin());
             clean(data.begin(), data.end());
+            munlock(data.data(), data.size()*sizeof(T));
         }
     }
 
@@ -575,8 +596,154 @@ private:
     std::vector<T> data_;
 };
 
+
+template <class T, size_t N>
+class OPENDHT_PUBLIC secure_array
+{
+public:
+    secure_array() {
+        mlock(data_.data(), data_.size()*sizeof(T));
+    }
+    explicit secure_array(T _item): data_( _item) {
+        mlock(data_.data(), data_.size()*sizeof(T));
+    }
+    explicit secure_array(const std::array<T,N>& c): data_(c) {
+        mlock(data_.data(), data_.size()*sizeof(T));
+    }
+    secure_array(secure_array<T,N> const&) = default;
+    secure_array(secure_array<T,N> &&) = delete;
+    ~secure_array() { clean(); munlock(data_.data(), data_.size()*sizeof(T)); }
+
+    static secure_array<T,N> getRandom() {
+        secure_array<T,N> ret;
+        crypto::random_device rdev;
+        std::uniform_int_distribution<uint8_t> rand_byte;
+        std::generate_n((uint8_t*)ret.data_.data(), ret.size()*sizeof(T), std::bind(rand_byte, std::ref(rdev)));
+        return ret;
+    }
+    secure_array<T,N>& operator=(const secure_array<T,N>& c) {
+        if (&c == this)
+            return *this;
+        clean();
+        data_ = c.data_;
+        return *this;
+    }
+    secure_array<T,N>& operator=(secure_array<T,N>&& c) = delete;
+    std::array<T,N>& writable() { clean(); return data_; }
+    const std::array<T,N>& makeInsecure() const { return data_; }
+    const uint8_t* data() const { return data_.data(); }
+    uint8_t* data() { return data_.data(); }
+
+    void clean() {
+        clean(data_.begin(), data_.end());
+    }
+    size_t size() const { return data_.size(); }
+
+private:
+    /**
+     * Securely wipe memory
+     */
+    static void clean(const typename std::array<T,N>::iterator& i, const typename std::array<T,N>::iterator& j) {
+        volatile uint8_t* b = reinterpret_cast<uint8_t*>(&*i);
+        volatile uint8_t* e = reinterpret_cast<uint8_t*>(&*j);
+        std::fill(b, e, 0);
+    }
+
+    std::array<T, N> data_;
+};
+
+
 using SecureBlob = secure_vector<uint8_t>;
 
+/*class EcPublicKey : public std::array<uint8_t, crypto_box_PUBLICKEYBYTES> {
+public:
+    Blob encrypt(const Blob& message) const {
+        Blob ret(crypto_box_SEALBYTES+message.size());
+        crypto_box_seal(ret.data(), message.data(), message.size(), data());
+        return ret;
+    }
+
+    const InfoHash& getId() const {
+        return *reinterpret_cast<const InfoHash*>(this);
+    }
+
+};*/
+
+class EcSecretKey {
+public:
+    using KeyData = secure_array<uint8_t, crypto_box_SECRETKEYBYTES>;
+    using EcPublicKey = InfoHash;
+
+    static EcSecretKey generate() {
+        EcSecretKey ret;
+        crypto_box_keypair(ret.pk.data(), ret.key.data());
+        return ret;
+    }
+
+    const EcPublicKey& getPublicKey() const {
+        return pk;
+    }
+
+    Blob encrypt(const uint8_t* data, size_t size, const EcPublicKey& pub) const {
+        Blob ret(crypto_box_NONCEBYTES+crypto_box_MACBYTES+size);
+        randombytes_buf(ret.data(), crypto_box_NONCEBYTES);
+        if (crypto_box_easy(ret.data()+crypto_box_NONCEBYTES, data, size, ret.data(),
+                            pub.data(), key.data()) != 0) {
+            throw CryptoException("Can't encrypt data");
+        }
+        return ret;
+    }
+
+    Blob decrypt(const uint8_t* cypher, size_t cypher_size) const {
+        if (cypher_size <= crypto_box_SEALBYTES)
+            throw DecryptError("Unexpected cipher length");
+        Blob ret(cypher_size - crypto_box_SEALBYTES);
+        if (crypto_box_seal_open(ret.data(), cypher, cypher_size, pk.data(), key.data()) != 0) {
+            throw DecryptError("Can't decrypt data");
+        }
+        return ret;
+    }
+
+    Blob decrypt(const uint8_t* cypher, size_t cypher_size, const EcPublicKey& pub) const {
+        if (cypher_size <= crypto_box_NONCEBYTES+crypto_box_MACBYTES)
+            throw DecryptError("Unexpected cipher length");
+        Blob ret(cypher_size - crypto_box_NONCEBYTES - crypto_box_MACBYTES);
+        if (crypto_box_open_easy(ret.data(),
+                cypher+crypto_box_NONCEBYTES,
+                cypher_size-crypto_box_NONCEBYTES,
+                cypher,
+                pub.data(), key.data()) != 0) {
+            throw DecryptError("Can't decrypt data");
+        }
+        return ret;
+    }
+
+    SecureBlob computeSecret(const EcPublicKey& pub, bool sending) const {
+        // shared key = h(q ‖ client_publickey ‖ server_publickey)
+        crypto_generichash_state h;
+        crypto_generichash_init(&h, nullptr, 0U, crypto_generichash_BYTES);
+        {
+            std::array<uint8_t, crypto_scalarmult_BYTES> q;
+            if (auto err = crypto_scalarmult(q.data(), key.data(), pub.data()))
+                throw CryptoException("Can't compute secret: " + std::to_string(err));
+            crypto_generichash_update(&h, q.data(), q.size());
+        }
+        {
+            const auto& sender_pk = sending ? pk : pub;
+            const auto& receiver_pk = sending ? pub : pk;
+            crypto_generichash_update(&h, sender_pk.data(), sender_pk.size());
+            crypto_generichash_update(&h, receiver_pk.data(), receiver_pk.size());
+        }
+        SecureBlob ret(crypto_generichash_BYTES);
+        crypto_generichash_final(&h, ret.data(), ret.size());
+        return ret;
+    }
+
+private:
+    KeyData key;
+    EcPublicKey pk;
+};
+
 /**
  * Generate an RSA key pair (4096 bits) and a certificate.
  * @param name the name used in the generated certificate
@@ -608,18 +775,18 @@ OPENDHT_PUBLIC Blob hash(const Blob& data, size_t hash_length = 512/8);
  * that can be transmitted in clear, and will be generated if
  * not provided (32 bytes).
  */
-OPENDHT_PUBLIC Blob stretchKey(const std::string& password, Blob& salt, size_t key_length = 512/8);
+OPENDHT_PUBLIC SecureBlob stretchKey(const std::string& password, Blob& salt, size_t key_length = 512/8);
 
 /**
  * AES-GCM encryption. Key must be 128, 192 or 256 bits long (16, 24 or 32 bytes).
  */
-OPENDHT_PUBLIC Blob aesEncrypt(const Blob& data, const Blob& key);
+OPENDHT_PUBLIC Blob aesEncrypt(const Blob& data, const SecureBlob& key);
 OPENDHT_PUBLIC Blob aesEncrypt(const Blob& data, const std::string& password);
 
 /**
  * AES-GCM decryption.
  */
-OPENDHT_PUBLIC Blob aesDecrypt(const Blob& data, const Blob& key);
+OPENDHT_PUBLIC Blob aesDecrypt(const Blob& data, const SecureBlob& key);
 OPENDHT_PUBLIC Blob aesDecrypt(const Blob& data, const std::string& password);
 
 }

include/opendht/dht.h

@@ -54,9 +54,6 @@ struct LocalListener;
 /**
  * Main Dht class.
  * Provides a Distributed Hash Table node.
- *
- * Must be given open UDP sockets and ::periodic must be
- * called regularly.
  */
 class OPENDHT_PUBLIC Dht {
 public:
@@ -73,13 +70,13 @@ public:
      * Initialise the Dht with two open sockets (for IPv4 and IP6)
      * and an ID for the node.
      */
-    Dht(int s, int s6, Config config);
+    Dht(uv_loop_t* loop, Config config);
     virtual ~Dht();
 
     /**
      * Get the ID of the node.
      */
-    inline const InfoHash& getNodeId() const { return myid; }
+    inline const InfoHash& getNodeId() const { return network_engine.getNodeId(); }
 
     /**
      * Get the current status of the node for the given family.
@@ -89,6 +86,9 @@ public:
     NodeStatus getStatus() const {
         return std::max(getStatus(AF_INET), getStatus(AF_INET6));
     }
+    void setOnStatusChanged(StatusCallback&& cb) {
+        statusCb = std::move(cb);
+    }
 
     /**
      * Performs final operations before quitting.
@@ -136,13 +136,13 @@ public:
         insertNode(n.id, SockAddr(n.ss, n.sslen));
     }
 
-    void pingNode(const sockaddr*, socklen_t, DoneCallbackSimple&& cb={});
-
-    time_point periodic(const uint8_t *buf, size_t buflen, const SockAddr&);
-    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) {
-        return periodic(buf, buflen, SockAddr(from, fromlen));
+    void bootstrap(const std::string& host, const std::string& service);
+    void clearBootstrap() {
+        bootstrap_nodes.clear();
     }
 
+    void pingNode(const SockAddr&, DoneCallback&& cb={});
+
     /**
      * Get a value by searching on all available protocols (IPv4, IPv6),
      * and call the provided get callback when values are found at key.
@@ -307,6 +307,10 @@ public:
 
     std::vector<SockAddr> getPublicAddress(sa_family_t family = 0);
 
+    SockAddr getBoundAddr() {
+        return network_engine.getBoundAddr();
+    }
+
 protected:
     Logger DHT_LOG;
     bool logFilerEnable_ {};
@@ -378,6 +382,8 @@ private:
     const bool is_bootstrap {false};
     const bool maintain_storage {false};
 
+    std::vector<std::pair<std::string,std::string>> bootstrap_nodes {};
+
     // the stuff
     RoutingTable buckets4 {};
     RoutingTable buckets6 {};
@@ -400,8 +406,7 @@ private:
 
     // timing
     Scheduler scheduler;
-    Sp<Scheduler::Job> nextNodesConfirmation {};
-    Sp<Scheduler::Job> nextStorageMaintenance {};
+    Sp<Job> nextNodesConfirmation {};
 
     net::NetworkEngine network_engine;
     unsigned pending_pings4 {0};
@@ -409,9 +414,13 @@ private:
 
     using ReportedAddr = std::pair<unsigned, SockAddr>;
     std::vector<ReportedAddr> reported_addr;
-
     std::mt19937_64 rd {crypto::getSeededRandomEngine<std::mt19937_64>()};
+    
+    NodeStatus status4 {NodeStatus::Disconnected},
+               status6 {NodeStatus::Disconnected};
+    StatusCallback statusCb {nullptr};
 
+    void updateStatus();
     void rotateSecrets();
 
     Blob makeToken(const SockAddr&, bool old) const;

include/opendht/dhtrunner.h

@@ -26,6 +26,7 @@
 #include "sockaddr.h"
 #include "log_enable.h"
 #include "def.h"
+#include "uv_utils.h"
 
 #include <thread>
 #include <mutex>
@@ -51,8 +52,6 @@ struct SecureDhtConfig;
 class OPENDHT_PUBLIC DhtRunner {
 
 public:
-    typedef std::function<void(NodeStatus, NodeStatus)> StatusCallback;
-
     DhtRunner();
     virtual ~DhtRunner();
 
@@ -202,8 +201,11 @@ public:
      * Insert known nodes to the routing table, without necessarly ping them.
      * Usefull to restart a node and get things running fast without putting load on the network.
      */
-    void bootstrap(const std::vector<std::pair<sockaddr_storage, socklen_t>>& nodes, DoneCallbackSimple&& cb={});
-    void bootstrap(const SockAddr& addr, DoneCallbackSimple&& cb={});
+    void ping(const std::vector<std::pair<sockaddr_storage, socklen_t>>& nodes, DoneCallback&& cb={});
+    void ping(const SockAddr& addr, DoneCallback&& cb={});
+    void ping(const SockAddr& addr, DoneCallbackSimple&& cb={}) {
+        ping(addr, bindDoneCb(cb));
+    }
 
     /**
      * Insert known nodes to the routing table, without necessarly ping them.
@@ -241,16 +243,14 @@ public:
      * Returns the currently bound address.
      * @param f: address family of the bound address to retreive.
      */
-    const SockAddr& getBound(sa_family_t f = AF_INET) const {
-        return (f == AF_INET) ? bound4 : bound6;
-    }
+    SockAddr getBound() const;
 
     /**
      * Returns the currently bound port, in host byte order.
      * @param f: address family of the bound port to retreive.
      */
-    in_port_t getBoundPort(sa_family_t f = AF_INET) const {
-        return getBound(f).getPort();
+    in_port_t getBoundPort() const {
+        return getBound().getPort();
     }
 
     std::pair<size_t, size_t> getStoreSize() const;
@@ -306,20 +306,24 @@ public:
      * @param cb: Optional callback to receive general state information.
      */
     void run(in_port_t port, const crypto::Identity identity, bool threaded = false, NetId network = 0) {
-        run(port, {
+        run({
             /*.dht_config = */{
                 /*.node_config = */{
                     /*.node_id = */{},
                     /*.network = */network,
                     /*.is_bootstrap = */false,
-                    /*.maintain_storage*/false
+                    /*.maintain_storage = */false,
+                    /* .network_config = */{
+                        /*.addr = */ "::0",
+                        /*.port = */ port
+                    }
                 },
                 /*.id = */identity
             },
             /*.threaded = */threaded
         });
     }
-    void run(in_port_t port, Config config);
+    void run(Config config);
 
     /**
      * @param local4: Local IPv4 address and port to bind. Can be null.
@@ -329,25 +333,22 @@ public:
      * @param threaded: If false, loop() must be called periodically. Otherwise a thread is launched.
      * @param cb: Optional callback to receive general state information.
      */
-    void run(const sockaddr_in* local4, const sockaddr_in6* local6, Config config);
+    //void run(const sockaddr_in* local4, const sockaddr_in6* local6, Config config);
 
     /**
      * Same as @run(sockaddr_in, sockaddr_in6, Identity, bool, StatusCallback), but with string IP addresses and service (port).
      */
-    void run(const char* ip4, const char* ip6, const char* service, Config config);
+    //void run(const char* ip4, const char* ip6, const char* service, Config config);
 
-    void setOnStatusChanged(StatusCallback&& cb) {
-        statusCb = std::move(cb);
-    }
+    void setOnStatusChanged(StatusCallback&& cb);
 
     /**
      * In non-threaded mode, the user should call this method
      * regularly and everytime a new packet is received.
      * @return the next op
      */
-    time_point loop() {
-        std::lock_guard<std::mutex> lck(dht_mtx);
-        return loop_();
+    void loop() {
+        uv_loop_.runOnce();
     }
 
     /**
@@ -373,33 +374,41 @@ private:
      */
     void tryBootstrapContinuously();
 
-    void doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDhtConfig config);
-    time_point loop_();
+    void doRun(SecureDhtConfig config);
+    void loop_();
 
     static std::vector<std::pair<sockaddr_storage, socklen_t>> getAddrInfo(const std::string& host, const std::string& service);
 
-    NodeStatus getStatus() const {
-        return std::max(status4, status6);
+    void loopCallback() {
+        std::lock_guard<std::mutex> lck(dht_mtx);
+        if (running)
+            loop_();
+        else {
+            uv_close((uv_handle_t*)&uv_async_, nullptr);
+            uv_loop_.stop();
+        }
+    }
+
+    static void loop_callback(uv_async_t* handle) {
+        if (handle->data)
+            static_cast<DhtRunner*>(handle->data)->loopCallback();
     }
 
     std::unique_ptr<SecureDht> dht_;
+    EventLoop uv_loop_;
+    uv_async_t uv_async_;
     mutable std::mutex dht_mtx {};
     std::thread dht_thread {};
-    std::condition_variable cv {};
-
-    std::thread rcv_thread {};
-    std::mutex sock_mtx {};
-    std::vector<std::pair<Blob, SockAddr>> rcv {};
 
     /** true if currently actively boostraping */
     std::atomic_bool bootstraping {false};
     /* bootstrap nodes given as (host, service) pairs */
-    std::vector<std::pair<std::string,std::string>> bootstrap_nodes_all {};
+    /*std::vector<std::pair<std::string,std::string>> bootstrap_nodes_all {};
     std::vector<std::pair<std::string,std::string>> bootstrap_nodes {};
-    std::thread bootstrap_thread {};
+    std::thread bootstrap_thread {};*/
     /** protects bootstrap_nodes, bootstrap_thread */
-    std::mutex bootstrap_mtx {};
-    std::condition_variable bootstrap_cv {};
+    /*std::mutex bootstrap_mtx {};
+    std::condition_variable bootstrap_cv {};*/
 
     std::queue<std::function<void(SecureDht&)>> pending_ops_prio {};
     std::queue<std::function<void(SecureDht&)>> pending_ops {};
@@ -407,12 +416,12 @@ private:
 
     std::atomic<bool> running {false};
 
-    NodeStatus status4 {NodeStatus::Disconnected},
+    /*NodeStatus status4 {NodeStatus::Disconnected},
                status6 {NodeStatus::Disconnected};
     StatusCallback statusCb {nullptr};
 
     SockAddr bound4 {};
-    SockAddr bound6 {};
+    SockAddr bound6 {};*/
 };
 
 }

include/opendht/infohash.h

@@ -22,6 +22,8 @@
 
 #include <msgpack.hpp>
 
+#include <sodium.h>
+
 #ifndef _WIN32
 #include <netinet/in.h>
 #include <netdb.h>
@@ -46,7 +48,7 @@ typedef uint16_t in_port_t;
 
 
 // bytes
-#define HASH_LEN 20u
+#define HASH_LEN crypto_box_PUBLICKEYBYTES
 
 namespace dht {
 
@@ -147,6 +149,12 @@ public:
             std::copy_n(h, HASH_LEN, begin());
     }
 
+    std::vector<uint8_t> encrypt(const uint8_t* msg, size_t msg_size) const {
+        std::vector<uint8_t> ret(crypto_box_SEALBYTES+msg_size);
+        crypto_box_seal(ret.data(), msg, msg_size, data());
+        return ret;
+    }
+
     /**
      * Constructor from an hexadecimal string (without "0x").
      * hex must be at least 2.HASH_LEN characters long.

include/opendht/network_engine.h

@@ -28,6 +28,8 @@
 #include "rng.h"
 #include "rate_limiter.h"
 #include "net.h"
+#include "callbacks.h"
+#include "uv_utils.h"
 
 #include <vector>
 #include <string>
@@ -202,8 +204,8 @@ public:
     using RequestCb = std::function<void(const Request&, RequestAnswer&&)>;
     using RequestExpiredCb = std::function<void(const Request&, bool)>;
 
-    NetworkEngine(Logger& log, Scheduler& scheduler);
-    NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& log, Scheduler& scheduler,
+    NetworkEngine(uv_loop_t*, Logger& log, Scheduler& scheduler);
+    NetworkEngine(uv_loop_t*, const NetworkConfig& config, NetId net, Logger& log, Scheduler& scheduler,
             decltype(NetworkEngine::onError) onError,
             decltype(NetworkEngine::onNewNode) onNewNode,
             decltype(NetworkEngine::onReportedAddr) onReportedAddr,
@@ -216,7 +218,16 @@ public:
 
     virtual ~NetworkEngine();
 
+    const InfoHash& getNodeId() const {
+        return id_key.getPublicKey();
+    }
+
     void clear();
+    void close(OnClose cb) {
+        sock->close(cb);
+        tcp_sock->close();
+        cache.closeAll();
+    }
 
     /**
      * Sends values (with closest nodes) to a listenner.
@@ -236,7 +247,7 @@ public:
             std::vector<Sp<Value>>&& values, const Query& q);
 
     bool isRunning(sa_family_t af) const;
-    inline want_t want () const { return dht_socket >= 0 && dht_socket6 >= 0 ? (WANT4 | WANT6) : -1; }
+    inline want_t want () const { /*return dht_socket >= 0 && dht_socket6 >= 0 ? (WANT4 | WANT6) : -1;*/return WANT4 | WANT6; }
 
     void connectivityChanged(sa_family_t);
 
@@ -254,7 +265,7 @@ public:
      * @return the request with information concerning its success.
      */
     Sp<Request>
-        sendPing(Sp<Node> n, RequestCb&& on_done, RequestExpiredCb&& on_expired);
+        sendPing(const Sp<Node>& n, RequestCb&& on_done, RequestExpiredCb&& on_expired);
     /**
      * Send a "ping" request to a given node.
      *
@@ -266,8 +277,8 @@ public:
      * @return the request with information concerning its success.
      */
     Sp<Request>
-        sendPing(const sockaddr* sa, socklen_t salen, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
-            return sendPing(std::make_shared<Node>(zeroes, sa, salen),
+        sendPing(const SockAddr& sa, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
+            return sendPing(std::make_shared<Node>(zeroes, sa),
                     std::forward<RequestCb>(on_done),
                     std::forward<RequestExpiredCb>(on_expired));
         }
@@ -283,7 +294,7 @@ public:
      *
      * @return the request with information concerning its success.
      */
-    Sp<Request> sendFindNode(Sp<Node> n,
+    Sp<Request> sendFindNode(const Sp<Node>& n,
                              const InfoHash& hash,
                              want_t want,
                              RequestCb&& on_done,
@@ -302,7 +313,7 @@ public:
      *
      * @return the request with information concerning its success.
      */
-    Sp<Request> sendGetValues(Sp<Node> n,
+    Sp<Request> sendGetValues(const Sp<Node>& n,
                               const InfoHash& hash,
                               const Query& query,
                               want_t want,
@@ -331,7 +342,7 @@ public:
      *
      * @return the request with information concerning its success.
      */
-    Sp<Request> sendListen(Sp<Node> n,
+    Sp<Request> sendListen(const Sp<Node>& n,
                            const InfoHash& hash,
                            const Query& query,
                            const Blob& token,
@@ -352,7 +363,7 @@ public:
      *
      * @return the request with information concerning its success.
      */
-    Sp<Request> sendAnnounceValue(Sp<Node> n,
+    Sp<Request> sendAnnounceValue(const Sp<Node>& n,
                                   const InfoHash& hash,
                                   const Sp<Value>& v,
                                   time_point created,
@@ -372,26 +383,15 @@ public:
      *
      * @return the request with information concerning its success.
      */
-    Sp<Request> sendRefreshValue(Sp<Node> n,
+    Sp<Request> sendRefreshValue(const Sp<Node>& n,
                                  const InfoHash& hash,
                                  const Value::Id& vid,
                                  const Blob& token,
                                  RequestCb&& on_done,
                                  RequestExpiredCb&& on_expired);
 
-    /**
-     * Parses a message and calls appropriate callbacks.
-     *
-     * @param buf  The buffer containing the binary message.
-     * @param buflen  The length of the buffer.
-     * @param from  The address info of the sender.
-     * @param fromlen  The length of the corresponding sockaddr structure.
-     * @param now  The time to adjust the clock in the network engine.
-     */
-    void processMessage(const uint8_t *buf, size_t buflen, const SockAddr& addr);
-
     Sp<Node> insertNode(const InfoHash& myid, const SockAddr& addr) {
-        auto n = cache.getNode(myid, addr, scheduler.time(), 0);
+        auto n = cache.getNode(myid, addr, {}, scheduler.time(), 0);
         onNewNode(n, 0);
         return n;
     }
@@ -409,6 +409,10 @@ public:
         return cache.getCachedNodes(id, sa_f, count);
     }
 
+    SockAddr getBoundAddr() {
+        return sock->getBoundAddr();
+    }
+
 private:
 
     struct PartialMessage;
@@ -437,7 +441,20 @@ private:
 
     static const std::string my_v;
 
-    void process(std::unique_ptr<ParsedMessage>&&, const SockAddr& from);
+    /**
+     * Parses a message and calls appropriate callbacks.
+     *
+     * @param buf  The buffer containing the binary message.
+     * @param buflen  The length of the buffer.
+     * @param from  The address info of the sender.
+     * @param fromlen  The length of the corresponding sockaddr structure.
+     * @param now  The time to adjust the clock in the network engine.
+     */
+    void processMessage(const uint8_t* data, size_t size, const SockAddr& addr);
+    Sp<Node> processEncrypted(const msgpack::object& o, const SockAddr& from);
+    Sp<Node> process(const msgpack::object& o, const SockAddr& addr);
+
+    Sp<Node> process(std::unique_ptr<ParsedMessage>&&, const SockAddr& from, const Sp<TcpSocket>& s = {});
 
     bool rateLimit(const SockAddr& addr);
 
@@ -462,20 +479,24 @@ private:
     };
 
 
-    // basic wrapper for socket sendto function
-    int send(const char *buf, size_t len, int flags, const SockAddr& addr);
+    int send(msgpack::sbuffer& msg, const Node& node, const Sp<TcpSocket>& sock = {});
+    int sendUDP(msgpack::sbuffer& msg, const SockAddr& addr);
+
+    int send(const Blob& msg, int flags, const Node& node, UdpSocket::OnSent&& cb);
+
+    void startTcp(const Sp<TcpSocket>& sock, const Sp<Node>& node = {});
 
     void sendValueParts(TransId tid, const std::vector<Blob>& svals, const SockAddr& addr);
-    std::vector<Blob> packValueHeader(msgpack::sbuffer&, const std::vector<Sp<Value>>&);
+    std::vector<Blob> packValueHeader(msgpack::sbuffer&, const std::vector<Sp<Value>>&, bool stream);
     void maintainRxBuffer(const TransId& tid);
 
     /*************
      *  Answers  *
      *************/
     /* answer to a ping  request */
-    void sendPong(const SockAddr& addr, TransId tid);
+    void sendPong(const Sp<Node>& node, TransId tid);
     /* answer to findnodes/getvalues request */
-    void sendNodesValues(const SockAddr& addr,
+    void sendNodesValues(const Sp<Node>& node,
             TransId tid,
             const Blob& nodes,
             const Blob& nodes6,
@@ -490,27 +511,32 @@ private:
             std::vector<Sp<Node>>& nodes,
             std::vector<Sp<Node>>& nodes6);
     /* answer to a listen request */
-    void sendListenConfirmation(const SockAddr& addr, TransId tid);
+    void sendListenConfirmation(const Sp<Node>& node, TransId tid);
     /* answer to put request */
-    void sendValueAnnounced(const SockAddr& addr, TransId, Value::Id);
+    void sendValueAnnounced(const Sp<Node>& node, TransId, Value::Id);
     /* answer in case of error */
-    void sendError(const SockAddr& addr,
+    void sendError(const Sp<Node>& node,
             TransId tid,
             uint16_t code,
             const std::string& message,
-            bool include_id=false);
+            bool include_id=false,
+            const Sp<TcpSocket>& sock= {});
 
     void deserializeNodes(ParsedMessage& msg, const SockAddr& from);
 
     /* DHT info */
-    const InfoHash& myid;
+    //const InfoHash& myid;
     const NetId network {0};
-    const int dht_socket {-1};
-    const int dht_socket6 {-1};
     const Logger& DHT_LOG;
 
+    Sp<UdpSocket> sock;
+    Sp<TcpSocket> tcp_sock;
+    std::set<Sp<TcpSocket>> pending_connect;
     NodeCache cache {};
 
+    crypto::EcSecretKey id_key;
+    //crypto::EcSecretKey id_key;
+
     // global limiting should be triggered by at least 8 different IPs
     using IpLimiter = RateLimiter<MAX_REQUESTS_PER_SEC/8>;
     using IpLimiterMap = std::map<SockAddr, IpLimiter, SockAddr::ipCmp>;

include/opendht/node.h

@@ -23,6 +23,8 @@
 #include "utils.h"
 #include "sockaddr.h"
 #include "net.h"
+#include "uv_utils.h"
+#include "crypto.h"
 
 #include <list>
 #include <map>
@@ -48,9 +50,18 @@ struct Socket {
 struct Node {
     const InfoHash id;
 
-    Node(const InfoHash& id, const SockAddr& addr, bool client=false);
+    Sp<TcpSocket> sock;
+
+    InfoHash last_known_pk;
+    //std::list<crypto::EcSecretKey>
+
+    Node(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& s = {}, bool client=false);
     Node(const InfoHash& id, const sockaddr* sa, socklen_t salen)
         : Node(id, SockAddr(sa, salen)) {}
+    ~Node() {
+        if (sock)
+            sock->close();
+    }
 
     InfoHash getId() const {
         return id;
@@ -59,6 +70,7 @@ struct Node {
     std::string getAddrStr() const {
         return addr.toString();
     }
+
     bool isClient() const { return is_client; }
     bool isIncoming() { return time > reply_time; }
 
@@ -66,6 +78,14 @@ struct Node {
     const time_point& getReplyTime() const { return reply_time; }
     void setTime(const time_point& t) { time = t; }
 
+    SockAddr getAddress() const {
+        return canStream() ? sock->getPeerAddr() : addr;
+    }
+
+    void startTCP() {
+
+    }
+
     /**
      * Makes notice about an additionnal authentication error with this node. Up
      * to MAX_AUTH_ERRORS errors are accepted in order to let the node recover.
@@ -98,7 +118,11 @@ struct Node {
     }
     sa_family_t getFamily() const { return addr.getFamily(); }
 
-    void update(const SockAddr&);
+    void update(const SockAddr&, const Sp<TcpSocket>&);
+
+    bool canStream() const {
+        return sock and not sock->isClosed() and sock->canWrite();
+    }
 
     void requested(const Sp<net::Request>& req);
     void received(time_point now, const Sp<net::Request>& req);

include/opendht/node_cache.h

@@ -27,7 +27,7 @@ namespace dht {
 
 struct NodeCache {
     Sp<Node> getNode(const InfoHash& id, sa_family_t family);
-    Sp<Node> getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client=false);
+    Sp<Node> getNode(const InfoHash& id, const SockAddr&, const Sp<TcpSocket>&, time_point now, bool confirmed, bool client=false);
     std::vector<Sp<Node>> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const;
 
     /**
@@ -36,13 +36,15 @@ struct NodeCache {
      * To use in case of connectivity change etc.
      */
     void clearBadNodes(sa_family_t family = 0);
+    void closeAll(sa_family_t family = 0);
 
 private:
     class NodeMap : public std::map<InfoHash, std::weak_ptr<Node>> {
     public:
         Sp<Node> getNode(const InfoHash& id);
-        Sp<Node> getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client);
+        Sp<Node> getNode(const InfoHash& id, const SockAddr&, const Sp<TcpSocket>&, time_point now, bool confirmed, bool client);
         void clearBadNodes();
+        void closeAll();
     };
 
     const NodeMap& cache(sa_family_t af) const { return af == AF_INET ? cache_4 : cache_6; }

include/opendht/routing_table.h

@@ -42,6 +42,14 @@ struct Bucket {
     Sp<Node> randomNode();
 
     void sendCachedPing(net::NetworkEngine& ne);
+
+    /** Return false if there is a single non-expired node in this bucket. */
+    bool expired() const {
+        for (const auto& n : nodes)
+            if (not n->isExpired())
+                return false;
+        return true;
+    }
 };
 
 class RoutingTable : public std::list<Bucket> {

include/opendht/scheduler.h

@@ -22,6 +22,9 @@
 
 #include "utils.h"
 #include "log_enable.h"
+#include "uv_utils.h"
+
+#include <uv.h>
 
 #include <functional>
 #include <map>
@@ -36,12 +39,7 @@ namespace dht {
  */
 class Scheduler {
 public:
-    Scheduler(const Logger& l) : DHT_LOG(l) {}
-
-    struct Job {
-        Job(std::function<void()>&& f) : do_(std::move(f)) {}
-        std::function<void()> do_;
-    };
+    Scheduler(uv_loop_t* loop, const Logger& l) : loop_(loop), DHT_LOG(l) {}
 
     /**
      * Adds another job to the queue.
@@ -51,10 +49,32 @@ public:
      *
      * @return pointer to the newly scheduled job.
      */
-    Sp<Scheduler::Job> add(time_point t, std::function<void()>&& job_func) {
-        auto job = std::make_shared<Job>(std::move(job_func));
-        if (t != time_point::max())
-            timers.emplace(std::move(t), job);
+    Sp<Job> add(const time_point& t, std::function<void()>&& job_func) {
+        if (not loop_)
+            return {};
+        auto job = Job::make(loop_, std::move(job_func));
+        if (t != time_point::max()) {
+            garbageCollection();
+            pending_jobs.emplace(job);
+            job->run(t);
+        } else
+            job->cancel();
+        return job;
+    }
+    Sp<Job> add(const duration& t, std::function<void()>&& job_func) {
+        if (not loop_)
+            return {};
+        auto job = Job::make(loop_, std::move(job_func));
+        garbageCollection();
+        pending_jobs.emplace(job);
+        job->run(t);
+        return job;
+    }
+    Sp<Job> run(std::function<void()>&& job_func) {
+        auto job = Job::make(loop_, std::move(job_func));
+        garbageCollection();
+        pending_jobs.emplace(job);
+        job->run();
         return job;
     }
 
@@ -66,7 +86,9 @@ public:
      *
      * @return pointer to the newly scheduled job.
      */
-    void edit(Sp<Scheduler::Job>& job, time_point t) {
+    void edit(Sp<Job>& job, const duration& t = {}) {
+        if (not loop_)
+            return;
         if (not job) {
             DHT_LOG.ERR("editing an empty job");
             return;
@@ -75,50 +97,73 @@ public:
         // Force clearing old value.
         auto task = std::move(job->do_);
         job->do_ = {};
+        job->cancel();
+        job = add(t, std::move(task));
+    }
+    void edit(Sp<Job>& job, const time_point& t) {
+        if (not loop_)
+            return;
+        if (not job) {
+            DHT_LOG.ERR("editing an empty job");
+            return;
+        }
+        auto task = std::move(job->do_);
+        job->do_ = {};
+        job->cancel();
         job = add(t, std::move(task));
     }
 
-    /**
-     * Runs the jobs to do up to now.
-     *
-     * @return The time for the next job to run.
-     */
-    time_point run() {
-        syncTime();
-        while (not timers.empty()) {
-            auto timer = timers.begin();
-            /*
-             * Running jobs scheduled before "now" prevents run+rescheduling
-             * loops before this method ends. It is garanteed by the fact that a
-             * job will at least be scheduled for "now" and not before.
-             */
-            if (timer->first > now)
-                break;
-
-            auto job = std::move(timer->second);
-            timers.erase(timer);
-
-            if (job->do_)
-                job->do_();
-        }
-        return getNextJobTime();
+    void stop() {
+        loop_ = nullptr;
+        size_t del {0};
+        for (auto& w : pending_jobs)
+            if (auto job = w.lock()) {
+                job->cancel();
+                del++;
+            }
+        pending_jobs.clear();
+        DHT_LOG.DEBUG("stopping scheduler, %lu events canceled", del);
     }
 
-    inline time_point getNextJobTime() const {
-        return timers.empty() ? time_point::max() : timers.begin()->first;
+    ~Scheduler(){
+        stop();
     }
 
     /**
      * Accessors for the common time reference used for synchronizing
      * operations.
      */
-    inline const time_point& time() const { return now; }
-    inline time_point syncTime() { return (now = clock::now()); }
+    inline time_point time() const {
+        return time_point(std::chrono::duration_cast<duration>(std::chrono::milliseconds(uv_now(loop_))));
+    }
+    inline time_point syncTime() {
+        return time();
+    }
+    uv_loop_t* getLoop() const {
+        return loop_;
+    }
 
 private:
-    time_point now {clock::now()};
-    std::multimap<time_point, Sp<Job>> timers {}; /* the jobs ordered by time */
+    std::set<std::weak_ptr<Job>, std::owner_less<std::weak_ptr<Job>>> pending_jobs {};
+    size_t lastSize_ {0};
+    uv_loop_t* loop_;
     const Logger& DHT_LOG;
+
+    void garbageCollection() {
+        if (pending_jobs.size() > lastSize_ * 2) {
+            unsigned del {0};
+            for (auto it = pending_jobs.begin(); it != pending_jobs.end();) {
+                if (it->expired()) {
+                    it = pending_jobs.erase(it);
+                    del++;
+                } else 
+                    ++it;
+            }
+            //DHT_LOG.DEBUG("pending_jobs cleanup: %u removed, %zu remaining", del, pending_jobs.size());
+            lastSize_ = pending_jobs.size();
+        }
+    }
+
 };
 
 }

include/opendht/securedht.h

@@ -44,7 +44,7 @@ public:
      * id:    the identity to use for the crypto layer and to compute
      *        our own hash on the Dht.
      */
-    SecureDht(int s, int s6, Config config);
+    SecureDht(uv_loop_t* loop, Config config);
 
     virtual ~SecureDht();
 

include/opendht/sockaddr.h

@@ -66,6 +66,19 @@ public:
             throw std::runtime_error("Socket address length is too large");
         set(sa, length);
     }
+    SockAddr(const sockaddr* sa) {
+        socklen_t len = 0;
+        if (sa) {
+            if (sa->sa_family == AF_INET)
+                len = sizeof(sockaddr_in);
+            else if(sa->sa_family == AF_INET6)
+                len = sizeof(sockaddr_in6);
+            else
+                throw std::runtime_error("Unknown address family");
+        }
+        set(sa, len);
+    }
+
     /**
      * Build from an existing sockaddr_storage structure.
      */
@@ -206,6 +219,9 @@ public:
 
     bool isUnspecified() const;
 
+    bool isMappedIPv4() const;
+    SockAddr getMappedIPv4() const;
+
     /**
      * A comparator to classify IP addresses, only considering the
      * first 64 bits in IPv6.

include/opendht/uv_utils.h

@@ -0,0 +1,419 @@
+#pragma once
+
+#include "utils.h"
+
+#include <uv.h>
+
+namespace dht {
+
+struct EventLoop {
+    using OnSignal = std::function<void(int)>;
+
+    EventLoop() {
+        uv_loop_init(&loop);
+        uv_signal_init(&loop, &sig_handler);
+        sig_handler.data = this;
+        uv_signal_start(&sig_handler, &EventLoop::signal_handler, SIGINT);
+    }
+    ~EventLoop() {
+        uv_loop_close(&loop);
+    }
+    auto run() {
+        return uv_run(&loop, UV_RUN_DEFAULT);
+    }
+    auto runOnce() {
+        return uv_run(&loop, UV_RUN_NOWAIT);
+    }
+    void stop() {
+        uv_signal_stop(&sig_handler);
+        uv_walk(&loop, &EventLoop::close_walk_cb, this);
+    }
+    uv_loop_t* get() { return &loop; }
+    void setOnSignal(OnSignal cb) {
+        onSignal_ = cb;
+    }
+private:
+    uv_loop_t loop;
+    uv_signal_t sig_handler;
+    OnSignal onSignal_;
+    void onSignal(int sig) {
+        if (onSignal_)
+            onSignal_(sig);
+    }
+    static void close_walk_cb(uv_handle_t* handle, void* /*arg*/) {
+        if (not uv_is_closing(handle))
+            uv_close(handle, [](uv_handle_t* handle) {
+                std::cout << "handle " << handle << " of type " << handle->type << " closed" << std::endl;
+            });
+    }
+    static void signal_handler(uv_signal_t* handle, int signum) {
+        if (handle->data) {
+            static_cast<EventLoop*>(handle->data)->onSignal(signum);
+        }
+    }
+};
+
+using GetAddrInfoCb = std::function<void(std::vector<SockAddr>&&)>;
+
+void getAddrInfo(uv_loop_t* loop, const char* node, const char* service, GetAddrInfoCb&& cb);
+
+struct Job : std::enable_shared_from_this<Job> {
+    Job(uv_loop_t* loop, std::function<void()>&& f = {}) : do_(std::move(f)) {
+        uv_timer_init(loop, &timer_);
+        timer_.data = nullptr;
+    }
+    void run(uint64_t t = 0) {
+        uv_timer_start(&timer_, &Job::timer_cb, t, 0);
+    }
+    inline void run(const duration& t) {
+        run(getTimeout(t));
+    }
+    inline void run(const time_point& t) {
+        run(getTimeout(t));
+    }
+    void cancel() {
+        if (timer_.data and not uv_is_closing((uv_handle_t*)&timer_)) {
+            uv_timer_stop(&timer_);
+            uv_close((uv_handle_t*)&timer_, &Job::close_cb);
+        }
+    }
+    static Sp<Job> make(uv_loop_t* loop, std::function<void()>&& f = {}) {
+        auto job = std::make_shared<Job>(loop, std::move(f));
+        job->timer_.data = new Sp<Job>(job);
+        return job;
+    }
+    static uint64_t getTimeout(const duration& d) {
+        return std::chrono::duration_cast<std::chrono::milliseconds>(d).count();
+    }
+    uint64_t getTimeout(const time_point& tp) const {
+        uint64_t t = getTimeout(tp.time_since_epoch());
+        uint64_t now = uv_now(timer_.loop);
+        return (t < now) ? 0 : t - now;
+    }
+    std::function<void()> do_;
+private:
+    uv_timer_t timer_;
+    void onTimer() {
+        if (do_)
+            do_();
+        cancel();
+    }
+    void onClosed() {
+        if (timer_.data) {
+            delete static_cast<Sp<Job>*>(timer_.data);
+            timer_.data = nullptr;
+        }
+    }
+    static void timer_cb(uv_timer_t* handle) {
+        if (handle->data) {
+            auto timer = *static_cast<Sp<Job>*>(handle->data);
+            timer->onTimer();
+        }
+    }
+    static void close_cb(uv_handle_t* handle) {
+        if (handle->data) {
+            auto timer = *static_cast<Sp<Job>*>(handle->data);
+            timer->onClosed();
+        } else {
+            std::cerr << "Job::onClosed(): no data! " << handle << std::endl;
+        }
+    }
+};
+
+
+static inline void alloc_buffer(uv_handle_t* /*handle*/, size_t suggested_size, uv_buf_t *buf) {
+    buf->base = (char*)malloc(suggested_size);
+    buf->len = suggested_size;
+}
+
+
+using OnClose = std::function<void()>;
+
+struct UdpSocket : public std::enable_shared_from_this<UdpSocket>
+{
+    using OnReceive = std::function<void(const uint8_t* data, size_t size, const SockAddr& addr)>;
+    using OnSent = std::function<void(int status)>;
+
+    UdpSocket(uv_loop_t* loop) {
+        uv_udp_init(loop, &sock);
+        sock.data = nullptr;
+    }
+    void open(const char* bind_addr, in_port_t port, OnReceive cb) {
+        if (not sock.data)
+            sock.data = new std::shared_ptr<UdpSocket>(shared_from_this());
+        receive_cb = std::move(cb);
+        struct sockaddr_in6 addr;
+        uv_ip6_addr(bind_addr, port, &addr);
+        uv_udp_bind(&sock, (const struct sockaddr*)&addr, 0);
+        uv_udp_recv_start(&sock, &UdpSocket::on_alloc_buffer, &UdpSocket::on_receive);
+    }
+
+    int send(uint8_t* data, size_t size, const SockAddr& addr, OnSent&& cb = {}) {
+        auto req = new SendReq;
+        req->msg = data;
+        req->req.data = req;
+        req->cb = std::move(cb);
+        const uv_buf_t buf {
+            /*.base =*/ (char*)data,
+            /*.len =*/ size
+        };
+        return uv_udp_send(&req->req, &sock, &buf, 1, addr.get(), &UdpSocket::on_sent);
+    }
+
+    void close(OnClose cb) {
+        close_cb = cb;
+        uv_close((uv_handle_t*) &sock, &UdpSocket::on_close);
+    }
+    bool isRunning() const {
+        return uv_is_active((uv_handle_t*) &sock);
+    }
+    SockAddr getBoundAddr() {
+        sockaddr_storage addr;
+        int size = sizeof(addr);
+        uv_udp_getsockname(&sock, (sockaddr*)&addr, &size);
+        return {(const sockaddr*)&addr, (socklen_t)size};
+    }
+private:
+    struct SendReq {
+        uv_udp_send_t req;
+        uint8_t* msg;
+        OnSent cb;
+    };
+
+    uv_udp_t sock;
+    OnReceive receive_cb;
+    OnClose close_cb;
+
+    void onAllocBuffer(size_t suggested_size, uv_buf_t* buf) {
+        buf->base = (char*)malloc(suggested_size);
+        buf->len = suggested_size;
+    }
+    void onReceive(ssize_t nread, const uv_buf_t* buf, const struct sockaddr* addr, unsigned /*flags*/) {
+        if (nread > 0 and addr and receive_cb) {
+            receive_cb((const uint8_t*)buf->base, nread, SockAddr(addr));
+        }
+        if (buf->base)
+            free(buf->base);
+    }
+    void onClosed() {
+        if (close_cb)
+            close_cb();
+        if (auto s = static_cast<std::shared_ptr<UdpSocket>*>(sock.data)) {
+            sock.data = nullptr;
+            delete s;
+        }
+    }
+    void onSent(SendReq& req, int status) {
+        if (req.cb) {
+            req.cb(status);
+        }
+        free(req.msg);
+    }
+    static std::shared_ptr<UdpSocket> get(uv_handle_t* handle) {
+        return handle->data ? *static_cast<std::shared_ptr<UdpSocket>*>(handle->data) : std::shared_ptr<UdpSocket>();
+    }
+    static void on_close(uv_handle_t* handle) {
+        if (auto node = get(handle))
+            node->onClosed();
+    }
+    static void on_sent(uv_udp_send_t* req, int status) {
+        if (auto data = static_cast<SendReq*>(req->data)) {
+            if (auto node = get((uv_handle_t*)req->handle))
+                node->onSent(*data, status);
+            delete data;
+        }
+    }
+    static void on_receive(uv_udp_t* handle, ssize_t nread, const uv_buf_t* buf, const struct sockaddr* addr, unsigned flags) {
+        if (auto node = get((uv_handle_t*)handle))
+            node->onReceive(nread, buf, addr, flags);
+    }
+    static inline void on_alloc_buffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t *buf) {
+        if (auto node = get((uv_handle_t*)handle))
+            node->onAllocBuffer(suggested_size, buf);
+    }
+};
+
+struct TcpSocket : public std::enable_shared_from_this<TcpSocket>
+{
+    using OnConnect = std::function<void(std::shared_ptr<TcpSocket>&& new_socket)>;
+    using OnConnected = std::function<void(int status)>;
+    using OnRead = std::function<void(const uint8_t* data, size_t size)>;
+    using OnWrite = std::function<void(int status)>;
+
+    TcpSocket(uv_loop_t* loop) {
+        uv_tcp_init(loop, &sock);
+        sock.data = nullptr;
+    }
+    void listen(const char* bind_addr, in_port_t port, OnConnect cb) {
+        if (not sock.data)
+            sock.data = new std::shared_ptr<TcpSocket>(shared_from_this());
+        connect_cb = cb;
+        struct sockaddr_in6 addr;
+        uv_ip6_addr(bind_addr, port, &addr);
+        uv_tcp_bind(&sock, (const struct sockaddr*)&addr, 0);
+        if (int r = uv_listen((uv_stream_t*)&sock, 128, &TcpSocket::on_new_connection)) {
+            fprintf(stderr, "Listen error %s\n", uv_strerror(r));
+        }
+    }
+    void connect(const sockaddr* addr, OnConnected&& cb) {
+        if (not sock.data)
+            sock.data = new std::shared_ptr<TcpSocket>(shared_from_this());
+        auto req = new ConnectReq;
+        req->connect.data = req;
+        req->sock = shared_from_this();
+        req->callback = std::move(cb);
+        uv_tcp_connect(&req->connect, &sock, addr, &TcpSocket::on_connect);
+    }
+    void start(OnRead&& cb) {
+        std::cerr << "TCP start" << std::endl;
+        read_cb = std::move(cb);
+        uv_read_start((uv_stream_t*)&sock, &TcpSocket::on_alloc_buffer, &TcpSocket::on_read);
+    }
+    int write(uint8_t* msg, size_t len, OnWrite&& cb = {}) {
+        std::cerr << "TCP write " << len << std::endl;
+        auto req = new WriteReq;
+        req->write.data = req;
+        req->msg = msg;
+        req->sock = shared_from_this();
+        req->callback = std::move(cb);
+        uv_buf_t buf;
+        buf.base = (char*)msg;
+        buf.len = len;
+        return uv_write(&req->write, (uv_stream_t *)&sock, &buf, 1, &TcpSocket::on_write_end);
+    }
+    void close(OnClose onclose = {}) {
+        std::cerr << "TCP close" << std::endl;
+        if (not isClosed()) {
+            close_cb = onclose;
+            uv_close((uv_handle_t*) &sock, &TcpSocket::on_close);
+        }
+    }
+    bool isClosed() {
+        return not sock.data or uv_is_closing((const uv_handle_t*) &sock);
+    }
+    bool canWrite() const {
+        return uv_is_writable((const uv_stream_t*)&sock);
+    }
+    SockAddr getBoundAddr() {
+        sockaddr_storage addr;
+        int size = sizeof(addr);
+        uv_tcp_getsockname(&sock, (sockaddr*)&addr, &size);
+        return {(const sockaddr*)&addr, (socklen_t)size};
+    }
+    SockAddr getPeerAddr() {
+        sockaddr_storage addr;
+        int size = sizeof(addr);
+        uv_tcp_getpeername(&sock, (sockaddr*)&addr, &size);
+        return {(const sockaddr*)&addr, (socklen_t)size};
+    }
+private:
+    uv_tcp_t sock;
+    OnConnect connect_cb;
+    OnRead read_cb;
+    OnClose close_cb;
+    struct ConnectReq {
+        uv_connect_t connect;
+        std::shared_ptr<TcpSocket> sock;
+        OnConnected callback;
+    };
+    struct WriteReq {
+        uv_write_t write;
+        uint8_t* msg;
+        std::shared_ptr<TcpSocket> sock;
+        OnWrite callback;
+    };
+    std::shared_ptr<TcpSocket> accept() {
+        auto s = std::make_shared<TcpSocket>(sock.loop);
+        s->sock.data = new std::shared_ptr<TcpSocket>(s);
+        if (uv_accept((uv_stream_t*)&sock, (uv_stream_t*) &s->sock) == 0) {
+            return s;
+        } else {
+            s->close();
+            return {};
+        }
+    }
+    void onNewConnection(int status) {
+        if (status < 0) {
+            fprintf(stderr, "New connection error %s\n", uv_strerror(status));
+            return;
+        }
+        if (auto s = accept())
+            connect_cb(std::move(s));
+        fprintf(stderr, "on_new_connection end\n");
+    }
+    void onConnected(int status, OnConnected&& cb) {
+        if (cb)
+            cb(status);
+    }
+    void onAllocBuffer(size_t suggested_size, uv_buf_t* buf) {
+        buf->base = (char*)malloc(suggested_size);
+        buf->len = suggested_size;
+    }
+    void onRead(ssize_t nread, const uv_buf_t *buf) {
+        if (nread < 0) {
+            if (nread != UV_EOF) {
+                fprintf(stderr, "Read error %s\n", uv_err_name(nread));
+            } else {
+                fprintf(stderr, "EOF\n");
+            }
+            close();
+        } else if (nread > 0) {
+            if (read_cb)
+                read_cb((const uint8_t*)buf->base, nread);
+        }
+        if (buf->base)
+            free(buf->base);
+    }
+    void onWrite(int status, OnWrite&& cb) {
+        if (cb)
+            cb(status);
+    }
+    void onClosed() {
+        read_cb = {};
+        connect_cb = {};
+        if (close_cb) {
+            close_cb();
+            close_cb = {};
+        }
+        if (auto s = static_cast<std::shared_ptr<TcpSocket>*>(sock.data)) {
+            sock.data = nullptr;
+            delete s;
+        }
+    }
+    static std::shared_ptr<TcpSocket> get(uv_handle_t* handle) {
+        return handle->data ? *static_cast<std::shared_ptr<TcpSocket>*>(handle->data) : std::shared_ptr<TcpSocket>();
+    }
+    static void on_new_connection(uv_stream_t* handle, int status) {
+        if (auto node = get((uv_handle_t*)handle))
+            node->onNewConnection(status);
+    }
+    static void on_close(uv_handle_t* handle) {
+        if (auto node = get(handle))
+            node->onClosed();
+    }
+    static inline void on_alloc_buffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t *buf) {
+        if (auto node = get((uv_handle_t*)handle))
+            node->onAllocBuffer(suggested_size, buf);
+    }
+    static void on_read(uv_stream_t* client, ssize_t nread, const uv_buf_t *buf) {
+        if (auto node = get((uv_handle_t*)client))
+            node->onRead(nread, buf);
+    }
+    static void on_write_end(uv_write_t* req, int status) {
+        if (auto write = static_cast<WriteReq*>(req->data)) {
+            write->sock->onWrite(status, std::move(write->callback));
+            if (write->msg)
+                free(write->msg);
+            delete write;
+        }
+    }
+    static void on_connect(uv_connect_t* req, int status) {
+        if (auto conn = static_cast<ConnectReq*>(req->data)) {
+            conn->sock->onConnected(status, std::move(conn->callback));
+            delete conn;
+        }
+    }
+};
+
+
+}

python/opendht.pyx

@@ -300,6 +300,8 @@ cdef class NodeSet(object):
         return s
     def __iter__(self):
         return NodeSetIter(self)
+    def __len__(self):
+        return self.size()
 
 cdef class PrivateKey(_WithID):
     cdef shared_ptr[cpp.PrivateKey] _key
@@ -442,6 +444,10 @@ cdef class DhtConfig(object):
         self._config.dht_config.node_config.node_id = id._infohash
     def setNetwork(self, unsigned netid):
         self._config.dht_config.node_config.network = netid
+    def setPort(self, cpp.in_port_t port):
+        self._config.dht_config.node_config.network_config.bind_port = port
+    def setBindAddr(self, string addr):
+        self._config.dht_config.node_config.network_config.bind_addr = addr
     def setMaintainStorage(self, bool maintain_storage):
         self._config.dht_config.node_config.maintain_storage = maintain_storage
 
@@ -460,7 +466,7 @@ cdef class DhtRunner(_WithID):
         if done_cb:
             cb_obj = {'done':done_cb}
             ref.Py_INCREF(cb_obj)
-            self.thisptr.get().bootstrap(addr._addr, cpp.bindDoneCbSimple(done_callback_simple, <void*>cb_obj))
+            self.thisptr.get().ping(addr._addr, cpp.bindDoneCbSimple(done_callback_simple, <void*>cb_obj))
         else:
             lock = threading.Condition()
             pending = 0
@@ -484,12 +490,10 @@ cdef class DhtRunner(_WithID):
     def run(self, Identity id=None, is_bootstrap=False, cpp.in_port_t port=0, str ipv4="", str ipv6="", DhtConfig config=DhtConfig()):
         if id:
             config.setIdentity(id)
-        if ipv4 or ipv6:
-            bind4 = ipv4.encode() if ipv4 else b''
-            bind6 = ipv6.encode() if ipv6 else b''
-            self.thisptr.get().run(bind4, bind6, str(port).encode(), config._config)
-        else:
-            self.thisptr.get().run(port, config._config)
+        config.setPort(port)
+        if ipv6:
+            config.setBindAddr(ipv6)
+        self.thisptr.get().run(config._config)
     def join(self):
         self.thisptr.get().join()
     def shutdown(self, shutdown_cb=None):
@@ -504,9 +508,9 @@ cdef class DhtRunner(_WithID):
         cpp.enableFileLogging(self.thisptr.get()[0], path.encode())
     def isRunning(self):
         return self.thisptr.get().isRunning()
-    def getBound(self, cpp.sa_family_t af = 0):
+    def getBound(self):
         s = SockAddr()
-        s._addr = self.thisptr.get().getBound(af)
+        s._addr = self.thisptr.get().getBound()
         return s
     def getStorageLog(self):
         return self.thisptr.get().getStorageLog().decode()

python/opendht_cpp.pxd

@@ -73,7 +73,7 @@ cdef extern from "opendht/sockaddr.h" namespace "dht":
         SockAddr() except +
         string toString() const
         in_port_t getPort() const
-        void setPort(in_port_t p)
+        void setPort(in_port_t p) const
         sa_family_t getFamily() const
         void setFamily(sa_family_t f)
 
@@ -198,11 +198,15 @@ cdef extern from "opendht/callbacks.h" namespace "dht":
     cdef DoneCallback bindDoneCb(DoneCallbackRaw cb, void *user_data)
     cdef DoneCallbackSimple bindDoneCbSimple(DoneCallbackSimpleRaw cb, void *user_data)
 
+    cppclass NetworkConfig:
+        string bind_addr
+        in_port_t bind_port
     cppclass Config:
         InfoHash node_id
         uint32_t network
         bool is_bootstrap
         bool maintain_storage
+        NetworkConfig network_config
     cppclass SecureDhtConfig:
         Config node_config
         Identity id
@@ -218,13 +222,12 @@ cdef extern from "opendht/dhtrunner.h" namespace "dht":
         InfoHash getId() const
         InfoHash getNodeId() const
         void bootstrap(const_char*, const_char*)
-        void bootstrap(const SockAddr&, DoneCallbackSimple done_cb)
-        void run(in_port_t, Config config)
-        void run(const_char*, const_char*, const_char*, Config config)
+        void ping(const SockAddr&, DoneCallbackSimple done_cb)
+        void run(Config config)
         void join()
         void shutdown(ShutdownCallback)
         bool isRunning()
-        SockAddr getBound(sa_family_t af) const
+        SockAddr getBound() const
         string getStorageLog() const
         string getRoutingTablesLog(sa_family_t af) const
         string getSearchesLog(sa_family_t af) const

python/setup.py.in

@@ -50,8 +50,8 @@ setup(name="opendht",
           ["@CURRENT_SOURCE_DIR@/opendht.pyx"],
           include_dirs = ['@PROJECT_SOURCE_DIR@/include'],
           language="c++",
-          extra_compile_args=["-std=c++11"],
-          extra_link_args=["-std=c++11"],
+          extra_compile_args=["-std=c++14"],
+          extra_link_args=["-std=c++14"],
           libraries=["opendht"],
           library_dirs = ['@CURRENT_BINARY_DIR@', '@PROJECT_BINARY_DIR@']
       ))

src/crypto.cpp

@@ -116,7 +116,7 @@ bool aesKeySizeGood(size_t key_size)
 #define GCM_DIGEST_SIZE GCM_BLOCK_SIZE
 #endif
 
-Blob aesEncrypt(const Blob& data, const Blob& key)
+Blob aesEncrypt(const Blob& data, const SecureBlob& key)
 {
     if (not aesKeySizeGood(key.size()))
         throw DecryptError("Wrong key size");
@@ -127,7 +127,7 @@ Blob aesEncrypt(const Blob& data, const Blob& key)
         std::generate_n(ret.begin(), GCM_IV_SIZE, std::bind(rand_byte, std::ref(rdev)));
     }
     struct gcm_aes_ctx aes;
-    gcm_aes_set_key(&aes, key.size(), key.data());
+    gcm_aes_set_key(&aes, key.size(), key.makeInsecure().data());
     gcm_aes_set_iv(&aes, GCM_IV_SIZE, ret.data());
     gcm_aes_update(&aes, data.size(), data.data());
 
@@ -139,13 +139,13 @@ Blob aesEncrypt(const Blob& data, const Blob& key)
 Blob aesEncrypt(const Blob& data, const std::string& password)
 {
     Blob salt;
-    Blob key = stretchKey(password, salt, 256 / 8);
+    SecureBlob key = stretchKey(password, salt, 256 / 8);
     Blob encrypted = aesEncrypt(data, key);
     encrypted.insert(encrypted.begin(), salt.begin(), salt.end());
     return encrypted;
 }
 
-Blob aesDecrypt(const Blob& data, const Blob& key)
+Blob aesDecrypt(const Blob& data, const SecureBlob& key)
 {
     if (not aesKeySizeGood(key.size()))
         throw DecryptError("Wrong key size");
@@ -185,12 +185,12 @@ Blob aesDecrypt(const Blob& data, const std::string& password)
     if (data.size() <= PASSWORD_SALT_LENGTH)
         throw DecryptError("Wrong data size");
     Blob salt {data.begin(), data.begin()+PASSWORD_SALT_LENGTH};
-    Blob key = stretchKey(password, salt, 256/8);
+    SecureBlob key = stretchKey(password, salt, 256/8);
     Blob encrypted {data.begin()+PASSWORD_SALT_LENGTH, data.end()};
     return aesDecrypt(encrypted, key);
 }
 
-Blob stretchKey(const std::string& password, Blob& salt, size_t key_length)
+SecureBlob stretchKey(const std::string& password, Blob& salt, size_t key_length)
 {
     if (salt.empty()) {
         salt.resize(PASSWORD_SALT_LENGTH);
@@ -506,11 +506,7 @@ PublicKey::encrypt(const Blob& data) const
     unsigned aes_key_sz = aesKeySize(max_block_sz);
     if (aes_key_sz == 0)
         throw CryptoException("Key is not long enough for AES128");
-    Blob key(aes_key_sz);
-    {
-        crypto::random_device rdev;
-        std::generate_n(key.begin(), key.size(), std::bind(rand_byte, std::ref(rdev)));
-    }
+    auto key = SecureBlob::getRandom(aes_key_sz);
     auto data_encrypted = aesEncrypt(data, key);
 
     Blob ret;
@@ -941,7 +937,7 @@ Certificate::generate(const PrivateKey& key, const std::string& name, Identity c
 
     if (ca.first && ca.second) {
         if (not ca.second->isCA()) {
-            // Signing certificate must be CA.
+            std::cerr << "Error: signing certificate must be CA." << std::endl;
             return {};
         }
         //if (gnutls_x509_crt_sign2(cert, ca.second->cert, ca.first->x509_key, get_dig(cert), 0) != GNUTLS_E_SUCCESS) {

src/dht.cpp

@@ -62,8 +62,16 @@ Dht::getStatus(sa_family_t af) const
 void
 Dht::shutdown(ShutdownCallback cb)
 {
+    auto final_shutdown = [this,cb](){
+        scheduler.stop();
+        DHT_LOG.w("closing socket...");
+        network_engine.close([=](){
+            if (cb)
+                cb();
+        });
+    };
     if (not maintain_storage) {
-        if (cb) cb();
+        final_shutdown();
         return;
     }
 
@@ -73,7 +81,7 @@ Dht::shutdown(ShutdownCallback cb)
     auto str_donecb = [=](bool, const std::vector<Sp<Node>>&) {
         --*remaining;
         DHT_LOG.w("shuting down node: %u ops remaining", *remaining);
-        if (!*remaining && cb) { cb(); }
+        if (!*remaining) { final_shutdown(); }
     };
 
     for (auto& str : store)
@@ -81,8 +89,7 @@ Dht::shutdown(ShutdownCallback cb)
 
     if (!*remaining) {
         DHT_LOG.w("shuting down node: %u ops remaining", *remaining);
-        if (cb)
-            cb();
+        final_shutdown();
     }
 }
 
@@ -528,7 +535,7 @@ Dht::searchStep(Sp<Search> sr)
         }
 
         // true if this node is part of the target nodes cluter.
-        /*bool in = sr->id.xorCmp(myid, sr->nodes.back().node->id) < 0;
+        /*bool in = sr->id.xorCmp(getNodeId(), sr->nodes.back().node->id) < 0;
 
         DHT_LOG_DEBUG("[search %s IPv%c] synced%s",
                 sr->id.toString().c_str(), sr->af == AF_INET ? '4' : '6', in ? ", in" : "");*/
@@ -700,7 +707,7 @@ Dht::search(const InfoHash& id, sa_family_t af, GetCallback gcb, QueryCallback q
     if (sr->nextSearchStep)
         scheduler.edit(sr->nextSearchStep, sr->getNextStepTime(scheduler.time()));
     else
-        sr->nextSearchStep = scheduler.add(scheduler.time(), std::bind(&Dht::searchStep, this, sr));
+        sr->nextSearchStep = scheduler.run(std::bind(&Dht::searchStep, this, sr));
 
     return sr;
 }
@@ -908,24 +915,24 @@ Dht::put(const InfoHash& id, Sp<Value> val, DoneCallback callback, time_point cr
     DHT_LOG.d(id, "put: adding %s -> %s", id.toString().c_str(), val->toString().c_str());
 
     auto op = std::make_shared<OpStatus>();
-    auto donecb = [callback](const std::vector<Sp<Node>>& nodes, OpStatus& op) {
+    auto donecb = [callback](std::vector<Sp<Node>>&& nodes, OpStatus& op) {
         // Callback as soon as the value is announced on one of the available networks
         if (callback and not op.status.done and (op.status4.done && op.status6.done)) {
-            callback(op.status4.ok or op.status6.ok, nodes);
+            callback(op.status4.ok or op.status6.ok, std::move(nodes));
             op.status.done = true;
         }
     };
-    announce(id, AF_INET, val, [=](bool ok4, const std::vector<Sp<Node>>& nodes) {
+    announce(id, AF_INET, val, [=](bool ok4, std::vector<Sp<Node>>&& nodes) {
         DHT_LOG.d(id, "Announce done IPv4 %d", ok4);
         auto& o = *op;
         o.status4 = {true, ok4};
-        donecb(nodes, o);
+        donecb(std::move(nodes), o);
     }, created, permanent);
-    announce(id, AF_INET6, val, [=](bool ok6, const std::vector<Sp<Node>>& nodes) {
+    announce(id, AF_INET6, val, [=](bool ok6, std::vector<Sp<Node>>&& nodes) {
         DHT_LOG.d(id, "Announce done IPv6 %d", ok6);
         auto& o = *op;
         o.status6 = {true, ok6};
-        donecb(nodes, o);
+        donecb(std::move(nodes), o);
     }, created, permanent);
 }
 
@@ -938,7 +945,7 @@ void doneCallbackWrapper(DoneCallback dcb, const std::vector<Sp<Node>>& nodes, G
         bool ok = op.status.ok or op.status4.ok or op.status6.ok;
         op.status.done = true;
         if (dcb)
-            dcb(ok, op.nodes);
+            dcb(ok, std::move(op.nodes));
     }
 }
 
@@ -961,6 +968,20 @@ bool callbackWrapper(Cb get_cb,
     return !op.status.ok;
 }
 
+
+void
+Dht::bootstrap(const std::string& host, const std::string& service)
+{
+    for (const auto& bn : bootstrap_nodes)
+        if (bn.first == host and bn.second == service)
+            return;
+    bootstrap_nodes.emplace_back(host, service);
+    getAddrInfo(scheduler.getLoop(), host.c_str(), service.c_str(), [this](std::vector<SockAddr>&& res) {
+        for (const auto& addr : res)
+            pingNode(addr);
+    });
+}
+
 void
 Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filter&& filter, Where&& where)
 {
@@ -988,15 +1009,15 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt
     /* Try to answer this search locally. */
     gcb(getLocal(id, f));
 
-    Dht::search(id, AF_INET, gcb, {}, [=](bool ok, const std::vector<Sp<Node>>& nodes) {
+    Dht::search(id, AF_INET, gcb, {}, [=](bool ok, std::vector<Sp<Node>>&& nodes) {
         //DHT_LOG_WARN("DHT done IPv4");
         op->status4 = {true, ok};
-        doneCallbackWrapper(donecb, nodes, *op);
+        doneCallbackWrapper(donecb, std::move(nodes), *op);
     }, f, q);
-    Dht::search(id, AF_INET6, gcb, {}, [=](bool ok, const std::vector<Sp<Node>>& nodes) {
+    Dht::search(id, AF_INET6, gcb, {}, [=](bool ok, std::vector<Sp<Node>>&& nodes) {
         //DHT_LOG_WARN("DHT done IPv6");
         op->status6 = {true, ok};
-        doneCallbackWrapper(donecb, nodes, *op);
+        doneCallbackWrapper(donecb, std::move(nodes), *op);
     }, f, q);
 }
 
@@ -1035,15 +1056,15 @@ void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Quer
     /* Try to answer this search locally. */
     qcb(local_fields);
 
-    Dht::search(id, AF_INET, {}, qcb, [=](bool ok, const std::vector<Sp<Node>>& nodes) {
+    Dht::search(id, AF_INET, {}, qcb, [=](bool ok, std::vector<Sp<Node>>&& nodes) {
         //DHT_LOG_WARN("DHT done IPv4");
         op->status4 = {true, ok};
-        doneCallbackWrapper(done_cb, nodes, *op);
+        doneCallbackWrapper(done_cb, std::move(nodes), *op);
     }, f, q);
-    Dht::search(id, AF_INET6, {}, qcb, [=](bool ok, const std::vector<Sp<Node>>& nodes) {
+    Dht::search(id, AF_INET6, {}, qcb, [=](bool ok, std::vector<Sp<Node>>&& nodes) {
         //DHT_LOG_WARN("DHT done IPv6");
         op->status6 = {true, ok};
-        doneCallbackWrapper(done_cb, nodes, *op);
+        doneCallbackWrapper(done_cb, std::move(nodes), *op);
     }, f, q);
 }
 
@@ -1300,6 +1321,7 @@ Dht::expireStore()
 void
 Dht::connectivityChanged(sa_family_t af)
 {
+    updateStatus();
     const auto& now = scheduler.time();
     scheduler.edit(nextNodesConfirmation, now);
     buckets(af).connectivityChanged(now);
@@ -1324,14 +1346,13 @@ Dht::rotateSecrets()
         secret = std::uniform_int_distribution<uint64_t>{}(rdev);
     }
     uniform_duration_distribution<> time_dist(std::chrono::minutes(15), std::chrono::minutes(45));
-    auto rotate_secrets_time = scheduler.time() + time_dist(rd);
-    scheduler.add(rotate_secrets_time, std::bind(&Dht::rotateSecrets, this));
+    scheduler.add(time_dist(rd), std::bind(&Dht::rotateSecrets, this));
 }
 
 Blob
 Dht::makeToken(const SockAddr& addr, bool old) const
 {
-    const void *ip;
+    const void* ip;
     size_t iplen;
     in_port_t port;
 
@@ -1340,12 +1361,12 @@ Dht::makeToken(const SockAddr& addr, bool old) const
         const auto& sin = addr.getIPv4();
         ip = &sin.sin_addr;
         iplen = 4;
-        port = sin.sin_port;
+        port = ntohs(sin.sin_port);
     } else if (family == AF_INET6) {
         const auto& sin6 = addr.getIPv6();
         ip = &sin6.sin6_addr;
         iplen = 16;
-        port = sin6.sin6_port;
+        port = ntohs(sin6.sin6_port);
     } else {
         return {};
     }
@@ -1389,7 +1410,7 @@ Dht::getNodesStats(sa_family_t af) const
         if (b.cached)
             stats.cached_nodes++;
     }
-    stats.table_depth = bcks.depth(bcks.findBucket(myid));
+    stats.table_depth = bcks.depth(bcks.findBucket(getNodeId()));
     return stats;
 }
 
@@ -1512,7 +1533,7 @@ void
 Dht::dumpTables() const
 {
     std::stringstream out;
-    out << "My id " << myid << std::endl;
+    out << "My id " << getNodeId() << std::endl;
 
     out << "Buckets IPv4 :" << std::endl;
     for (const auto& b : buckets4)
@@ -1653,14 +1674,13 @@ Dht::~Dht()
         s.second->clear();
 }
 
-Dht::Dht() : store(), scheduler(DHT_LOG), network_engine(DHT_LOG, scheduler) {}
+Dht::Dht() : store(), scheduler(nullptr, DHT_LOG), network_engine(nullptr, DHT_LOG, scheduler) {}
 
-Dht::Dht(int s, int s6, Config config)
-    : myid(config.node_id != zeroes ? config.node_id : InfoHash::getRandom()),
-    is_bootstrap(config.is_bootstrap),
-    maintain_storage(config.maintain_storage), store(), store_quota(),
-    scheduler(DHT_LOG),
-    network_engine(myid, config.network, s, s6, DHT_LOG, scheduler,
+Dht::Dht(uv_loop_t* loop, Config config)
+    : is_bootstrap(config.is_bootstrap), maintain_storage(config.maintain_storage),
+    buckets4{Bucket {AF_INET}},buckets6{Bucket {AF_INET6}}, store(), store_quota(),
+    scheduler(loop, DHT_LOG),
+    network_engine(loop, config.network_config, config.network, DHT_LOG, scheduler,
             std::bind(&Dht::onError, this, _1, _2),
             std::bind(&Dht::onNewNode, this, _1, _2),
             std::bind(&Dht::onReportedAddr, this, _1, _2),
@@ -1672,23 +1692,12 @@ Dht::Dht(int s, int s6, Config config)
             std::bind(&Dht::onRefresh, this, _1, _2, _3, _4))
 {
     scheduler.syncTime();
-    if (s < 0 && s6 < 0)
-        return;
-    if (s >= 0) {
-        buckets4 = {Bucket {AF_INET}};
-        buckets4.is_client = config.is_bootstrap;
-    }
-    if (s6 >= 0) {
-        buckets6 = {Bucket {AF_INET6}};
-        buckets6.is_client = config.is_bootstrap;
-    }
-
+    buckets4.is_client = config.is_bootstrap;
+    buckets6.is_client = config.is_bootstrap;
     search_id = std::uniform_int_distribution<decltype(search_id)>{}(rd);
 
     uniform_duration_distribution<> time_dis {std::chrono::seconds(3), std::chrono::seconds(5)};
-    auto confirm_nodes_time = scheduler.time() + time_dis(rd);
-    DHT_LOG.d(myid, "Scheduling %s", myid.toString().c_str());
-    nextNodesConfirmation = scheduler.add(confirm_nodes_time, std::bind(&Dht::confirmNodes, this));
+    nextNodesConfirmation = scheduler.add(time_dis(rd), std::bind(&Dht::confirmNodes, this));
 
     // Fill old secret
     {
@@ -1699,7 +1708,7 @@ Dht::Dht(int s, int s6, Config config)
 
     expire();
 
-    DHT_LOG.d("DHT initialised with node ID %s", myid.toString().c_str());
+    DHT_LOG.d("DHT initialised with node ID %s", getNodeId().toString().c_str());
 }
 
 
@@ -1707,11 +1716,11 @@ bool
 Dht::neighbourhoodMaintenance(RoutingTable& list)
 {
     //DHT_LOG_DEBUG("neighbourhoodMaintenance");
-    auto b = list.findBucket(myid);
+    auto b = list.findBucket(getNodeId());
     if (b == list.end())
         return false;
 
-    InfoHash id = myid;
+    InfoHash id = getNodeId();
 #ifdef _WIN32
     std::uniform_int_distribution<int> rand_byte{ 0, std::numeric_limits<uint8_t>::max() };
 #else
@@ -1735,7 +1744,13 @@ Dht::neighbourhoodMaintenance(RoutingTable& list)
                 n->toString().c_str(), id.toString().c_str());
         /* Since our node-id is the same in both DHTs, it's probably
            profitable to query both families. */
-        network_engine.sendFindNode(n, id, network_engine.want(), nullptr, nullptr);
+        network_engine.sendFindNode(n, id, network_engine.want(), nullptr, [this,n](const net::Request&, bool over) {
+            if (over) {
+                DHT_LOG.d(n->id, "[node %s] neighborhood maintenance op expired", n->toString().c_str());
+                scheduler.edit(nextNodesConfirmation, Node::MAX_RESPONSE_TIME);
+                updateStatus();
+            }
+        });
     }
 
     return true;
@@ -1749,7 +1764,7 @@ Dht::bucketMaintenance(RoutingTable& list)
 
     bool sent {false};
     for (auto b = list.begin(); b != list.end(); ++b) {
-        if (b->time < scheduler.time() - std::chrono::minutes(10) || b->nodes.empty()) {
+        if (b->time < scheduler.time() - std::chrono::minutes(10) || b->expired()) {
             /* This bucket hasn't seen any positive confirmation for a long
                time. Pick a random id in this bucket's range, and send a request
                to a random node. */
@@ -1758,11 +1773,11 @@ Dht::bucketMaintenance(RoutingTable& list)
             /* If the bucket is empty, we try to fill it from a neighbour.
                We also sometimes do it gratuitiously to recover from
                buckets full of broken nodes. */
-            if (std::next(b) != list.end() && (q->nodes.empty() || rand_trial(rd)))
+            if (std::next(b) != list.end() && (q->expired() || rand_trial(rd)))
                 q = std::next(b);
-            if (b != list.begin() && (q->nodes.empty() || rand_trial(rd))) {
+            if (b != list.begin() && (q->expired() || rand_trial(rd))) {
                 auto r = std::prev(b);
-                if (!r->nodes.empty())
+                if (!r->expired())
                     q = r;
             }
 
@@ -1772,26 +1787,23 @@ Dht::bucketMaintenance(RoutingTable& list)
 
                 if (network_engine.want() != want) {
                     auto otherbucket = findBucket(id, q->af == AF_INET ? AF_INET6 : AF_INET);
-                    if (otherbucket && otherbucket->nodes.size() < TARGET_NODES)
+                    if ((otherbucket and otherbucket->nodes.size() < TARGET_NODES) or rand_trial_38(rd))
                         /* The corresponding bucket in the other family
                            is emptyish -- querying both is useful. */
                         want = WANT4 | WANT6;
-                    else if (rand_trial_38(rd))
                         /* Most of the time, this just adds overhead.
                            However, it might help stitch back one of
                            the DHTs after a network collapse, so query
                            both, but only very occasionally. */
-                        want = WANT4 | WANT6;
                 }
 
                 DHT_LOG.d(id, n->id, "[node %s] sending find %s for bucket maintenance", n->toString().c_str(), id.toString().c_str());
                 auto start = scheduler.time();
                 network_engine.sendFindNode(n, id, want, nullptr, [this,start,n](const net::Request&, bool over) {
                     if (over) {
-                        const auto& end = scheduler.time();
-                        using namespace std::chrono;
-                        DHT_LOG.d(n->id, "[node %s] bucket maintenance op expired after %llu ms", n->toString().c_str(), duration_cast<milliseconds>(end-start).count());
-                        scheduler.edit(nextNodesConfirmation, end + Node::MAX_RESPONSE_TIME);
+                        DHT_LOG.d(n->id, "[node %s] bucket maintenance op expired", n->toString().c_str());
+                        scheduler.edit(nextNodesConfirmation, Node::MAX_RESPONSE_TIME);
+                        updateStatus();
                     }
                 });
                 sent = true;
@@ -1825,7 +1837,7 @@ Dht::maintainStorage(decltype(store)::value_type& storage, bool force, DoneCallb
 
     auto nodes = buckets4.findClosestNodes(storage.first, now);
     if (!nodes.empty()) {
-        if (force || storage.first.xorCmp(nodes.back()->id, myid) < 0) {
+        if (force || storage.first.xorCmp(nodes.back()->id, getNodeId()) < 0) {
             for (auto &value : storage.second.getValues()) {
                 const auto& vt = getType(value.data->type);
                 if (force || value.created + vt.expiration > now + MAX_STORAGE_MAINTENANCE_EXPIRE_TIME) {
@@ -1840,7 +1852,7 @@ Dht::maintainStorage(decltype(store)::value_type& storage, bool force, DoneCallb
 
     auto nodes6 = buckets6.findClosestNodes(storage.first, now);
     if (!nodes6.empty()) {
-        if (force || storage.first.xorCmp(nodes6.back()->id, myid) < 0) {
+        if (force || storage.first.xorCmp(nodes6.back()->id, getNodeId()) < 0) {
             for (auto &value : storage.second.getValues()) {
                 const auto& vt = getType(value.data->type);
                 if (force || value.created + vt.expiration > now + MAX_STORAGE_MAINTENANCE_EXPIRE_TIME) {
@@ -1863,7 +1875,7 @@ Dht::maintainStorage(decltype(store)::value_type& storage, bool force, DoneCallb
     return announce_per_af;
 }
 
-time_point
+/*time_point
 Dht::periodic(const uint8_t *buf, size_t buflen, const SockAddr& from)
 {
     scheduler.syncTime();
@@ -1875,19 +1887,37 @@ Dht::periodic(const uint8_t *buf, size_t buflen, const SockAddr& from)
         }
     }
     return scheduler.run();
+}*/
+
+void
+Dht::updateStatus()
+{
+    NodeStatus nstatus4 = getStatus(AF_INET);
+    NodeStatus nstatus6 = getStatus(AF_INET6);
+    if (nstatus4 != status4 || nstatus6 != status6) {
+        status4 = nstatus4;
+        status6 = nstatus6;
+        if (status4 == NodeStatus::Disconnected and status6 == NodeStatus::Disconnected) {
+            // We have lost connection with the DHT.  Try to recover using bootstrap nodes.
+            DHT_LOG.e(getNodeId(), "DHT disconnected", getNodeId().toString().c_str());
+        } else {
+            //bootstrap_nodes.clear();
+        }
+        if (statusCb)
+            statusCb(status4, status6);
+    }
 }
 
 void
 Dht::expire()
 {
     uniform_duration_distribution<> time_dis(std::chrono::minutes(2), std::chrono::minutes(6));
-    auto expire_stuff_time = scheduler.time() + duration(time_dis(rd));
 
     expireBuckets(buckets4);
     expireBuckets(buckets6);
     expireStore();
     expireSearches();
-    scheduler.add(expire_stuff_time, std::bind(&Dht::expire, this));
+    scheduler.add(time_dis(rd), std::bind(&Dht::expire, this));
 }
 
 void
@@ -1897,13 +1927,15 @@ Dht::confirmNodes()
     bool soon = false;
     const auto& now = scheduler.time();
 
-    if (searches4.empty() and getStatus(AF_INET) == NodeStatus::Connected) {
-        DHT_LOG.d(myid, "[confirm nodes] initial IPv4 'get' for my id (%s)", myid.toString().c_str());
-        search(myid, AF_INET);
+    updateStatus();
+
+    if (searches4.empty() and status4 == NodeStatus::Connected) {
+        DHT_LOG.d(getNodeId(), "[confirm nodes] initial IPv4 'get' for my id (%s)", getNodeId().toString().c_str());
+        search(getNodeId(), AF_INET);
     }
-    if (searches6.empty() and getStatus(AF_INET6) == NodeStatus::Connected) {
-        DHT_LOG.d(myid, "[confirm nodes] initial IPv6 'get' for my id (%s)", myid.toString().c_str());
-        search(myid, AF_INET6);
+    if (searches6.empty() and status6 == NodeStatus::Connected) {
+        DHT_LOG.d(getNodeId(), "[confirm nodes] initial IPv6 'get' for my id (%s)", getNodeId().toString().c_str());
+        search(getNodeId(), AF_INET6);
     }
 
     soon |= bucketMaintenance(buckets4);
@@ -1923,9 +1955,7 @@ Dht::confirmNodes()
     auto time_dis = soon
         ? uniform_duration_distribution<> {seconds(5) , seconds(25)}
         : uniform_duration_distribution<> {seconds(60), seconds(180)};
-    auto confirm_nodes_time = now + time_dis(rd);
-
-    scheduler.edit(nextNodesConfirmation, confirm_nodes_time);
+    scheduler.edit(nextNodesConfirmation, time_dis(rd));
 }
 
 std::vector<ValuesExport>
@@ -1995,13 +2025,13 @@ Dht::exportNodes()
 {
     const auto& now = scheduler.time();
     std::vector<NodeExport> nodes;
-    const auto b4 = buckets4.findBucket(myid);
+    const auto b4 = buckets4.findBucket(getNodeId());
     if (b4 != buckets4.end()) {
         for (auto& n : b4->nodes)
             if (n->isGood(now))
                 nodes.push_back(n->exportNode());
     }
-    const auto b6 = buckets6.findBucket(myid);
+    const auto b6 = buckets6.findBucket(getNodeId());
     if (b6 != buckets6.end()) {
         for (auto& n : b6->nodes)
             if (n->isGood(now))
@@ -2032,21 +2062,21 @@ Dht::insertNode(const InfoHash& id, const SockAddr& addr)
 }
 
 void
-Dht::pingNode(const sockaddr* sa, socklen_t salen, DoneCallbackSimple&& cb)
+Dht::pingNode(const SockAddr& sa, DoneCallback&& cb)
 {
     scheduler.syncTime();
-    DHT_LOG.d("Sending ping to %s", print_addr(sa, salen).c_str());
-    auto& count = sa->sa_family == AF_INET ? pending_pings4 : pending_pings6;
+    DHT_LOG.d("Sending ping to %s", sa.toString().c_str());
+    auto& count = sa.getFamily() == AF_INET ? pending_pings4 : pending_pings6;
     count++;
-    network_engine.sendPing(sa, salen, [&count,cb](const net::Request&, net::RequestAnswer&&) {
+    network_engine.sendPing(sa, [&count,cb](const net::Request& req, net::RequestAnswer&&) {
         count--;
         if (cb)
-            cb(true);
+            cb(true, {req.node});
     }, [&count,cb](const net::Request&, bool last){
         if (last) {
             count--;
             if (cb)
-                cb(false);
+                cb(false, {});
         }
     });
 }
@@ -2207,7 +2237,7 @@ Dht::onListen(Sp<Node> node, const InfoHash& hash, const Blob& token, size_t soc
             net::DhtProtocolException::LISTEN_NO_INFOHASH
         };
     }
-    if (!tokenMatch(token, node->getAddr())) {
+    if (not tokenMatch(token, node->getAddress())) {
         DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'listen'", node->toString().c_str(), hash.toString().c_str());
         throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::LISTEN_WRONG_TOKEN};
     }
@@ -2246,7 +2276,7 @@ Dht::onAnnounce(Sp<Node> n,
             net::DhtProtocolException::PUT_NO_INFOHASH
         };
     }
-    if (!tokenMatch(token, node.getAddr())) {
+    if (not tokenMatch(token, node.getAddr())) {
         DHT_LOG.w(hash, node.id, "[node %s] incorrect token %s for 'put'", node.toString().c_str(), hash.toString().c_str());
         throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::PUT_WRONG_TOKEN};
     }
@@ -2254,7 +2284,7 @@ Dht::onAnnounce(Sp<Node> n,
         // We store a value only if we think we're part of the
         // SEARCH_NODES nodes around the target id.
         auto closest_nodes = buckets(node.getFamily()).findClosestNodes(hash, scheduler.time(), SEARCH_NODES);
-        if (closest_nodes.size() >= TARGET_NODES and hash.xorCmp(closest_nodes.back()->id, myid) < 0) {
+        if (closest_nodes.size() >= TARGET_NODES and hash.xorCmp(closest_nodes.back()->id, getNodeId()) < 0) {
             DHT_LOG.w(hash, node.id, "[node %s] announce too far from the target. Dropping value.", node.toString().c_str());
             return {};
         }

src/dhtrunner.cpp

@@ -20,98 +20,32 @@
 #include "dhtrunner.h"
 #include "securedht.h"
 
-#ifndef _WIN32
-#include <unistd.h>
-#else
-#include <io.h>
-#endif
-
-#ifndef _WIN32
-#include <sys/socket.h>
-#else
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#define close(x) closesocket(x)
-#endif
-
 namespace dht {
 
 constexpr std::chrono::seconds DhtRunner::BOOTSTRAP_PERIOD;
 
 DhtRunner::DhtRunner() : dht_()
 {
-#ifdef _WIN32
-    WSADATA wsd;
-    if (WSAStartup(MAKEWORD(2,2), &wsd) != 0)
-        throw DhtException("Can't initialize Winsock2");
-#endif
+    uv_async_.data = this;
+    uv_async_init(uv_loop_.get(), &uv_async_, &DhtRunner::loop_callback);
 }
 
 DhtRunner::~DhtRunner()
 {
     join();
-#ifdef _WIN32
-    WSACleanup();
-#endif
 }
 
 void
-DhtRunner::run(in_port_t port, DhtRunner::Config config)
-{
-    sockaddr_in sin4;
-    std::fill_n((uint8_t*)&sin4, sizeof(sin4), 0);
-    sin4.sin_family = AF_INET;
-    sin4.sin_port = htons(port);
-    sockaddr_in6 sin6;
-    std::fill_n((uint8_t*)&sin6, sizeof(sin6), 0);
-    sin6.sin6_family = AF_INET6;
-    sin6.sin6_port = htons(port);
-    run(&sin4, &sin6, config);
-}
-
-void
-DhtRunner::run(const char* ip4, const char* ip6, const char* service, DhtRunner::Config config)
-{
-    auto res4 = getAddrInfo(ip4, service);
-    auto res6 = getAddrInfo(ip6, service);
-    run(res4.empty() ? nullptr : (sockaddr_in*) &res4.front().first,
-        res6.empty() ? nullptr : (sockaddr_in6*)&res6.front().first, config);
-}
-
-void
-DhtRunner::run(const sockaddr_in* local4, const sockaddr_in6* local6, DhtRunner::Config config)
+DhtRunner::run(DhtRunner::Config config)
 {
     if (running)
         return;
-    if (rcv_thread.joinable())
-        rcv_thread.join();
     running = true;
-    doRun(local4, local6, config.dht_config);
+    doRun(config.dht_config);
     if (not config.threaded)
         return;
     dht_thread = std::thread([this]() {
-        while (running) {
-            std::unique_lock<std::mutex> lk(dht_mtx);
-            auto wakeup = loop_();
-            cv.wait_until(lk, wakeup, [this]() {
-                if (not running)
-                    return true;
-                {
-                    std::lock_guard<std::mutex> lck(sock_mtx);
-                    if (not rcv.empty())
-                        return true;
-                }
-                {
-                    std::lock_guard<std::mutex> lck(storage_mtx);
-                    if (not pending_ops_prio.empty())
-                        return true;
-                    auto s = getStatus();
-                    if (not pending_ops.empty() and (s == NodeStatus::Connected or (s == NodeStatus::Disconnected and not bootstraping)))
-                        return true;
-                }
-                return false;
-            });
-        }
+        uv_loop_.run();
     });
 }
 
@@ -121,21 +55,19 @@ DhtRunner::shutdown(ShutdownCallback cb) {
     pending_ops_prio.emplace([=](SecureDht& dht) mutable {
         dht.shutdown(cb);
     });
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
 DhtRunner::join()
 {
     running = false;
-    cv.notify_all();
-    bootstrap_cv.notify_all();
+    uv_async_send(&uv_async_);
+    //bootstrap_cv.notify_all();
     if (dht_thread.joinable())
         dht_thread.join();
-    if (rcv_thread.joinable())
-        rcv_thread.join();
-    if (bootstrap_thread.joinable())
-        bootstrap_thread.join();
+    /*if (bootstrap_thread.joinable())
+        bootstrap_thread.join();*/
 
     {
         std::lock_guard<std::mutex> lck(storage_mtx);
@@ -145,10 +77,10 @@ DhtRunner::join()
     {
         std::lock_guard<std::mutex> lck(dht_mtx);
         dht_.reset();
-        status4 = NodeStatus::Disconnected;
+        /*status4 = NodeStatus::Disconnected;
         status6 = NodeStatus::Disconnected;
         bound4 = {};
-        bound6 = {};
+        bound6 = {};*/
     }
 }
 
@@ -175,6 +107,14 @@ DhtRunner::getNodeId() const
     return dht_->getNodeId();
 }
 
+void
+DhtRunner::setOnStatusChanged(StatusCallback&& cb)
+{
+    std::lock_guard<std::mutex> lck(dht_mtx);
+    if (not dht_)
+        throw std::runtime_error("dht is not running");
+    dht_->setOnStatusChanged(std::move(cb));
+}
 
 std::pair<size_t, size_t>
 DhtRunner::getStoreSize() const {
@@ -307,6 +247,11 @@ DhtRunner::getPublicAddressStr(sa_family_t af)
     return ret;
 }
 
+SockAddr 
+DhtRunner::getBound() const {
+    return dht_->getBoundAddr();
+}
+
 void
 DhtRunner::registerCertificate(std::shared_ptr<crypto::Certificate> cert) {
     std::lock_guard<std::mutex> lck(dht_mtx);
@@ -318,42 +263,12 @@ DhtRunner::setLocalCertificateStore(CertificateStoreQuery&& query_method) {
     dht_->setLocalCertificateStore(std::forward<CertificateStoreQuery>(query_method));
 }
 
-time_point
+void
 DhtRunner::loop_()
 {
     if (!dht_)
-        return {};
-
-    decltype(pending_ops) ops {};
-    {
-        std::lock_guard<std::mutex> lck(storage_mtx);
-        auto s = getStatus();
-        ops = (pending_ops_prio.empty() && (s == NodeStatus::Connected or (s == NodeStatus::Disconnected and not bootstraping))) ?
-               std::move(pending_ops) : std::move(pending_ops_prio);
-    }
-    while (not ops.empty()) {
-        ops.front()(*dht_);
-        ops.pop();
-    }
-
-    time_point wakeup {};
-    decltype(rcv) received {};
-    {
-        std::lock_guard<std::mutex> lck(sock_mtx);
-        // move to stack
-        received = std::move(rcv);
-    }
-    if (not received.empty()) {
-        for (const auto& pck : received) {
-            auto& buf = pck.first;
-            auto& from = pck.second;
-            wakeup = dht_->periodic(buf.data(), buf.size()-1, from);
-        }
-        received.clear();
-    } else {
-        wakeup = dht_->periodic(nullptr, 0, nullptr, 0);
-    }
-
+        return;
+/*
     NodeStatus nstatus4 = dht_->getStatus(AF_INET);
     NodeStatus nstatus6 = dht_->getStatus(AF_INET6);
     if (nstatus4 != status4 || nstatus6 != status6) {
@@ -371,106 +286,24 @@ DhtRunner::loop_()
         if (statusCb)
             statusCb(status4, status6);
     }
-
-    return wakeup;
+*/
+    decltype(pending_ops) ops {};
+    {
+        std::lock_guard<std::mutex> lck(storage_mtx);
+        auto s = dht_->getStatus();
+        ops = (pending_ops_prio.empty() && (s == NodeStatus::Connected or (s == NodeStatus::Disconnected and not bootstraping))) ?
+               std::move(pending_ops) : std::move(pending_ops_prio);
+    }
+    while (not ops.empty()) {
+        ops.front()(*dht_);
+        ops.pop();
+    }
 }
 
 void
-DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::Config config)
+DhtRunner::doRun(SecureDht::Config config)
 {
-    dht_.reset();
-
-    int s4 = -1,
-        s6 = -1;
-
-    bound4 = {};
-    if (sin4) {
-        s4 = socket(PF_INET, SOCK_DGRAM, 0);
-        if(s4 >= 0) {
-            int rc = bind(s4, (sockaddr*)sin4, sizeof(sockaddr_in));
-            if(rc < 0)
-                throw DhtException("Can't bind IPv4 socket on " + dht::print_addr((sockaddr*)sin4, sizeof(sockaddr_in)));
-            sockaddr_storage ss;
-            socklen_t ss_len = sizeof(ss);
-            getsockname(s4, (sockaddr*)&ss, &ss_len);
-            bound4 = {ss, ss_len};
-        }
-    }
-
-#if 1
-    bound6 = {};
-    if (sin6) {
-        s6 = socket(PF_INET6, SOCK_DGRAM, 0);
-        if(s6 >= 0) {
-            int val = 1;
-            int rc = setsockopt(s6, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&val, sizeof(val));
-            if(rc < 0)
-                throw DhtException("Can't set IPV6_V6ONLY");
-
-            rc = bind(s6, (sockaddr*)sin6, sizeof(sockaddr_in6));
-            if(rc < 0)
-                throw DhtException("Can't bind IPv6 socket on " + dht::print_addr((sockaddr*)sin6, sizeof(sockaddr_in6)));
-            sockaddr_storage ss;
-            socklen_t ss_len = sizeof(ss);
-            getsockname(s6, (sockaddr*)&ss, &ss_len);
-            bound6 = {ss, ss_len};
-        }
-    }
-#endif
-
-    dht_ = std::unique_ptr<SecureDht>(new SecureDht {s4, s6, config});
-
-    rcv_thread = std::thread([this,s4,s6]() {
-        try {
-            while (true) {
-                struct timeval tv {/*.tv_sec = */0, /*.tv_usec = */250000};
-                fd_set readfds;
-
-                FD_ZERO(&readfds);
-                if(s4 >= 0)
-                    FD_SET(s4, &readfds);
-                if(s6 >= 0)
-                    FD_SET(s6, &readfds);
-
-                int rc = select(s4 > s6 ? s4 + 1 : s6 + 1, &readfds, nullptr, nullptr, &tv);
-                if(rc < 0) {
-                    if(errno != EINTR) {
-                        perror("select");
-                        std::this_thread::sleep_for( std::chrono::seconds(1) );
-                    }
-                }
-
-                if(!running)
-                    break;
-
-                if(rc > 0) {
-                    std::array<uint8_t, 1024 * 64> buf;
-                    sockaddr_storage from;
-                    socklen_t from_len = sizeof(from);
-
-                    if(s4 >= 0 && FD_ISSET(s4, &readfds))
-                        rc = recvfrom(s4, (char*)buf.data(), buf.size(), 0, (sockaddr*)&from, &from_len);
-                    else if(s6 >= 0 && FD_ISSET(s6, &readfds))
-                        rc = recvfrom(s6, (char*)buf.data(), buf.size(), 0, (sockaddr*)&from, &from_len);
-                    else
-                        break;
-                    if (rc > 0) {
-                        {
-                            std::lock_guard<std::mutex> lck(sock_mtx);
-                            rcv.emplace_back(Blob {buf.begin(), buf.begin()+rc+1}, SockAddr(from, from_len));
-                        }
-                        cv.notify_all();
-                    }
-                }
-            }
-        } catch (const std::exception& e) {
-            std::cerr << "Error in DHT networking thread: " << e.what() << std::endl;
-        }
-        if (s4 >= 0)
-            close(s4);
-        if (s6 >= 0)
-            close(s6);
-    });
+    dht_ = std::unique_ptr<SecureDht>(new SecureDht {uv_loop_.get(), config});
 }
 
 void
@@ -479,10 +312,11 @@ DhtRunner::get(InfoHash hash, GetCallback vcb, DoneCallback dcb, Value::Filter f
     {
         std::lock_guard<std::mutex> lck(storage_mtx);
         pending_ops.emplace([=](SecureDht& dht) mutable {
+            std::cout << "DhtRunner dht.get " << hash << std::endl;
             dht.get(hash, vcb, dcb, std::move(f), std::move(w));
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -497,7 +331,7 @@ void DhtRunner::query(const InfoHash& hash, QueryCallback cb, DoneCallback done_
             dht.query(hash, cb, done_cb, std::move(q));
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 std::future<size_t>
@@ -510,7 +344,7 @@ DhtRunner::listen(InfoHash hash, GetCallback vcb, Value::Filter f, Where w)
             ret_token->set_value(dht.listen(hash, vcb, std::move(f), std::move(w)));
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
     return ret_token->get_future();
 }
 
@@ -529,7 +363,7 @@ DhtRunner::cancelListen(InfoHash h, size_t token)
             dht.cancelListen(h, token);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -542,7 +376,7 @@ DhtRunner::cancelListen(InfoHash h, std::shared_future<size_t> token)
             dht.cancelListen(h, tk);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -555,7 +389,7 @@ DhtRunner::put(InfoHash hash, Value&& value, DoneCallback cb, time_point created
             dht.put(hash, sv, cb, created, permanent);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -567,7 +401,7 @@ DhtRunner::put(InfoHash hash, std::shared_ptr<Value> value, DoneCallback cb, tim
             dht.put(hash, value, cb, created, permanent);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -585,7 +419,7 @@ DhtRunner::cancelPut(const InfoHash& h , const Value::Id& id)
             dht.cancelPut(h, id);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -597,7 +431,7 @@ DhtRunner::putSigned(InfoHash hash, std::shared_ptr<Value> value, DoneCallback c
             dht.putSigned(hash, value, cb);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -621,7 +455,7 @@ DhtRunner::putEncrypted(InfoHash hash, InfoHash to, std::shared_ptr<Value> value
             dht.putEncrypted(hash, to, value, cb);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -636,6 +470,7 @@ DhtRunner::putEncrypted(const std::string& key, InfoHash to, Value&& value, Done
     putEncrypted(InfoHash::get(key), to, std::forward<Value>(value), cb);
 }
 
+/*
 void
 DhtRunner::tryBootstrapContinuously()
 {
@@ -693,7 +528,7 @@ DhtRunner::tryBootstrapContinuously()
             }
         } while (bootstraping);
     });
-}
+}*/
 
 std::vector<std::pair<sockaddr_storage, socklen_t>>
 DhtRunner::getAddrInfo(const std::string& host, const std::string& service)
@@ -723,45 +558,60 @@ DhtRunner::getAddrInfo(const std::string& host, const std::string& service)
 void
 DhtRunner::bootstrap(const std::string& host, const std::string& service)
 {
-    std::lock_guard<std::mutex> lck(bootstrap_mtx);
+    /*std::lock_guard<std::mutex> lck(bootstrap_mtx);
     bootstrap_nodes_all.emplace_back(host, service);
     bootstrap_nodes.emplace_back(host, service);
-    tryBootstrapContinuously();
+    tryBootstrapContinuously();*/
+    pending_ops_prio.emplace([=](SecureDht& dht) mutable {
+        dht.bootstrap(host, service);
+    });
+    uv_async_send(&uv_async_);
 }
 
 void
 DhtRunner::clearBootstrap()
 {
-    std::lock_guard<std::mutex> lck(bootstrap_mtx);
-    bootstrap_nodes_all.clear();
+    //std::lock_guard<std::mutex> lck(bootstrap_mtx);
+    //bootstrap_nodes_all.clear();
+    pending_ops_prio.emplace([=](SecureDht& dht) mutable {
+        dht.clearBootstrap();
+    });
+    uv_async_send(&uv_async_);
 }
 
 void
-DhtRunner::bootstrap(const std::vector<std::pair<sockaddr_storage, socklen_t>>& nodes, DoneCallbackSimple&& cb)
+DhtRunner::ping(const std::vector<std::pair<sockaddr_storage, socklen_t>>& nodes, DoneCallback&& cb)
 {
     std::lock_guard<std::mutex> lck(storage_mtx);
     pending_ops_prio.emplace([=](SecureDht& dht) mutable {
-        auto rem = cb ? std::make_shared<std::pair<size_t, bool>>(nodes.size(), false) : nullptr;
+        struct GroupPing {
+            size_t remaining {0};
+            bool ok {false};
+            std::vector<Sp<Node>> nodes;
+            GroupPing(size_t s) : remaining(s) {}
+        };
+        Sp<GroupPing> rem = cb ? std::make_shared<GroupPing>(nodes.size()) : nullptr;
         for (auto& node : nodes)
-            dht.pingNode((sockaddr*)&node.first, node.second, cb ? [rem,cb](bool ok) {
+            dht.pingNode(SockAddr((sockaddr*)&node.first, node.second), cb ? [rem,cb](bool ok, std::vector<Sp<Node>>&& nodes) {
                 auto& r = *rem;
-                r.first--;
-                r.second |= ok;
-                if (not r.first)
-                    cb(r.second);
-            } : DoneCallbackSimple{});
+                r.remaining--;
+                r.ok |= ok;
+                r.nodes.insert(r.nodes.end(), nodes.begin(), nodes.end());
+                if (r.remaining == 0)
+                    cb(r.ok, std::move(r.nodes));
+            } : DoneCallback{});
     });
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
-DhtRunner::bootstrap(const SockAddr& addr, DoneCallbackSimple&& cb)
+DhtRunner::ping(const SockAddr& addr, DoneCallback&& cb)
 {
     std::lock_guard<std::mutex> lck(storage_mtx);
     pending_ops_prio.emplace([addr,cb](SecureDht& dht) mutable {
-        dht.pingNode(addr.get(), addr.getLength(), std::move(cb));
+        dht.pingNode(addr, std::move(cb));
     });
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -774,7 +624,7 @@ DhtRunner::bootstrap(const std::vector<NodeExport>& nodes)
                 dht.insertNode(node);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -786,7 +636,7 @@ DhtRunner::connectivityChanged()
             dht.connectivityChanged();
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 void
@@ -797,7 +647,7 @@ DhtRunner::findCertificate(InfoHash hash, std::function<void(const std::shared_p
             dht.findCertificate(hash, cb);
         });
     }
-    cv.notify_all();
+    uv_async_send(&uv_async_);
 }
 
 }

src/network_engine.cpp

@@ -69,32 +69,6 @@ static const uint8_t v4prefix[16] = {
 
 constexpr unsigned SEND_NODES {8};
 
-#ifdef _WIN32
-
-static bool
-set_nonblocking(int fd, int nonblocking)
-{
-    unsigned long mode = !!nonblocking;
-    int rc = ioctlsocket(fd, FIONBIO, &mode);
-    return rc == 0;
-}
-
-extern const char *inet_ntop(int, const void *, char *, socklen_t);
-
-#else
-
-static bool
-set_nonblocking(int fd, int nonblocking)
-{
-    int rc = fcntl(fd, F_GETFL, 0);
-    if (rc < 0)
-        return false;
-    rc = fcntl(fd, F_SETFL, nonblocking?(rc | O_NONBLOCK):(rc & ~O_NONBLOCK));
-    return rc >= 0;
-}
-
-#endif
-
 struct NetworkEngine::PartialMessage {
     SockAddr from;
     time_point start;
@@ -116,8 +90,8 @@ RequestAnswer::RequestAnswer(ParsedMessage&& msg)
  : ntoken(std::move(msg.token)), values(std::move(msg.values)), fields(std::move(msg.fields)),
     nodes4(std::move(msg.nodes4)), nodes6(std::move(msg.nodes6)) {}
 
-NetworkEngine::NetworkEngine(Logger& log, Scheduler& scheduler) : myid(zeroes), DHT_LOG(log), scheduler(scheduler) {}
-NetworkEngine::NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& log, Scheduler& scheduler,
+NetworkEngine::NetworkEngine(uv_loop_t*, Logger& log, Scheduler& scheduler) : DHT_LOG(log), scheduler(scheduler) {}
+NetworkEngine::NetworkEngine(uv_loop_t* loop, const NetworkConfig& config, NetId net, Logger& log, Scheduler& scheduler,
         decltype(NetworkEngine::onError) onError,
         decltype(NetworkEngine::onNewNode) onNewNode,
         decltype(NetworkEngine::onReportedAddr) onReportedAddr,
@@ -128,21 +102,18 @@ NetworkEngine::NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& l
         decltype(NetworkEngine::onAnnounce) onAnnounce,
         decltype(NetworkEngine::onRefresh) onRefresh) :
     onError(onError), onNewNode(onNewNode), onReportedAddr(onReportedAddr), onPing(onPing), onFindNode(onFindNode),
-    onGetValues(onGetValues), onListen(onListen), onAnnounce(onAnnounce), onRefresh(onRefresh), myid(myid),
-    network(net), dht_socket(s), dht_socket6(s6), DHT_LOG(log), scheduler(scheduler)
+    onGetValues(onGetValues), onListen(onListen), onAnnounce(onAnnounce), onRefresh(onRefresh),
+    network(net), DHT_LOG(log), sock(std::make_shared<UdpSocket>(loop)), tcp_sock(std::make_shared<TcpSocket>(loop)),
+    id_key(crypto::EcSecretKey::generate()), scheduler(scheduler)
 {
-    if (dht_socket >= 0) {
-        if (!set_nonblocking(dht_socket, 1))
-            throw DhtException("Can't set socket to non-blocking mode");
-    }
-    if (dht_socket6 >= 0) {
-        if (!set_nonblocking(dht_socket6, 1))
-            throw DhtException("Can't set socket to non-blocking mode");
-    }
+    using namespace std::placeholders;
+    sock->open(config.bind_addr.c_str(), config.bind_port, std::bind(&NetworkEngine::processMessage, this, _1, _2, _3));
+    tcp_sock->listen(config.bind_addr.c_str(), config.bind_port, [&](Sp<TcpSocket>&& new_socket){
+        startTcp(new_socket);
+    });
 }
 
 NetworkEngine::~NetworkEngine() {
-    clear();
 }
 
 void
@@ -153,25 +124,16 @@ NetworkEngine::tellListener(Sp<Node> node, uint32_t socket_id, const InfoHash& h
 {
     auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6);
     try {
-        sendNodesValues(node->getAddr(), TransId(socket_id), nnodes.first, nnodes.second, values, query, ntoken);
+        sendNodesValues(node, TransId(socket_id), nnodes.first, nnodes.second, values, query, ntoken);
     } catch (const std::overflow_error& e) {
         DHT_LOG.e("Can't send value: buffer not large enough !");
     }
 }
 
 bool
-NetworkEngine::isRunning(sa_family_t af) const
+NetworkEngine::isRunning(sa_family_t /*af*/) const
 {
-    switch (af) {
-    case 0:
-        return dht_socket  >= 0 ||  dht_socket6 >= 0;
-    case AF_INET:
-        return dht_socket  >= 0;
-    case AF_INET6:
-        return dht_socket6 >= 0;
-    default:
-        return false;
-    }
+    return sock->isRunning();
 }
 
 void
@@ -203,27 +165,41 @@ NetworkEngine::requestStep(Sp<Request> sreq)
         req.on_expired(req, false);
     }
 
-    auto err = send((char*)req.msg.data(), req.msg.size(),
-            (node.getReplyTime() >= now - UDP_REPLY_TIME) ? 0 : MSG_CONFIRM,
-            node.getAddr());
-    if (err == ENETUNREACH  ||
-        err == EHOSTUNREACH ||
-        err == EAFNOSUPPORT)
-    {
-        node.setExpired();
-        if (node.id == zeroes)
-            requests.erase(req.tid);
-    } else {
-        if (err != EAGAIN) {
-            ++req.attempt_count;
-        }
+    if (node.sock and node.sock->canWrite()) {
+        auto data = (uint8_t*)malloc(req.msg.size());
+        memcpy(data, req.msg.data(), req.msg.size());
         req.last_try = now;
-        std::weak_ptr<Request> wreq = sreq;
-        scheduler.add(req.last_try + Node::MAX_RESPONSE_TIME, [this,wreq] {
-            if (auto req = wreq.lock())
-                requestStep(req);
+        ++req.attempt_count;
+        node.sock->write(data, req.msg.size(), [&,sreq](int status){
+            if (status != 0) {
+                DHT_LOG.w(sreq->node->id, "write failed ! %d", status);
+                sreq->node->setExpired();
+            }
         });
+        return;
     }
+    // UDP
+    std::weak_ptr<Request> wreq = sreq;
+    send(req.msg, 0, node, [this,wreq](int status) {
+        if (auto req = wreq.lock()) {
+            if (status >= 0) {
+                ++req->attempt_count;
+                return;
+            }
+            if (status == -EAGAIN)
+                return;
+            auto& node = *req->node;
+            DHT_LOG.e(node.id, "[node %s] can't send packet: %d %s", node.toString().c_str(), status, uv_strerror(status));
+            node.setExpired();
+            if (node.id == zeroes)
+                requests.erase(req->tid);
+        }
+    });
+    req.last_try = now;
+    scheduler.add(req.last_try + Node::MAX_RESPONSE_TIME, [this,wreq] {
+        if (auto req = wreq.lock())
+            requestStep(req);
+    });
 }
 
 /**
@@ -307,8 +283,86 @@ NetworkEngine::isNodeBlacklisted(const SockAddr& addr) const
 }
 
 void
-NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr& from)
+NetworkEngine::startTcp(const Sp<TcpSocket>& sock, const Sp<Node>& node)
 {
+    struct SockRxData {
+        msgpack::unpacker unpacker;
+        /** the socket have been assigned to a node */
+        Sp<Node> node;
+    };
+
+    auto rx_data = std::make_shared<SockRxData>();
+    rx_data->node = node;
+    if (not rx_data->node)
+        pending_connect.emplace(sock);
+    //std::cout << "startTcp " << rx_data->assigned << std::endl;
+    //sock->start(std::bind(&NetworkEngine::onReceiveData, this, _1, _2, n, sock));
+    sock->start([this,sock,rx_data](const uint8_t* data, size_t size){
+        std::cout << "Received message !! " << size << std::endl;
+        rx_data->unpacker.reserve_buffer(size);
+        memcpy(rx_data->unpacker.buffer(), data, size);
+        rx_data->unpacker.buffer_consumed(size);
+        msgpack::object_handle result;
+        // Message pack data loop
+        while(rx_data->unpacker.next(result)) {
+            std::unique_ptr<ParsedMessage> msg {new ParsedMessage};
+            SockAddr from = sock->getPeerAddr();
+            if (from.isMappedIPv4())
+                from = from.getMappedIPv4();
+            Sp<Node> node_found;
+            try {
+                const auto& o = result.get();
+                if (o.type == msgpack::type::BIN) {
+                    auto decrypted = id_key.decrypt((const uint8_t*)o.via.bin.ptr, o.via.bin.size);
+                    msgpack::object_handle oh = msgpack::unpack((const char *)decrypted.data(), decrypted.size());
+                    //msg->msgpack_unpack(oh.get());
+                    node_found = processEncrypted(oh.get(), from);
+                } else {
+                    msg->msgpack_unpack(o);
+                    node_found = process(std::move(msg), from, sock);
+                }
+            } catch (const std::exception& e) {
+                DHT_LOG.w("Can't process message: %s", e.what());
+                return;
+            }
+            if (node_found) {
+                if (not rx_data->node) {
+                    pending_connect.erase(sock);
+                    rx_data->node = node_found;
+                } else if (rx_data->node != node_found) {
+                    DHT_LOG.w("Changing node id on TCP socket !");
+                }
+            }
+            /*if (msg->network != network) {
+                DHT_LOG.d("Received message from other network %u", msg->network);
+                return;
+            }
+            SockAddr from = sock->getPeerAddr();
+            if (from.isMappedIPv4())
+                from = from.getMappedIPv4();
+            process(std::move(msg), from, sock);*/
+        }
+    });
+}
+
+Sp<Node>
+NetworkEngine::processEncrypted(const msgpack::object& o, const SockAddr& from)
+{
+    if (o.type != msgpack::type::ARRAY)
+        throw msgpack::type_error();
+    InfoHash id {o.via.array.ptr[0]};
+    const auto& data = o.via.array.ptr[1];
+    if (data.type != msgpack::type::BIN)
+        throw msgpack::type_error();
+    auto decrypted = id_key.decrypt((const uint8_t *)data.via.bin.ptr, data.via.bin.size, id);
+    msgpack::object_handle oh = msgpack::unpack((const char *)decrypted.data(), decrypted.size());
+    return process(oh.get(), from);
+}
+
+void
+NetworkEngine::processMessage(const uint8_t* buf, size_t buflen, const SockAddr& from_raw)
+{
+    SockAddr from = from_raw.isMappedIPv4() ? from_raw.getMappedIPv4() : from_raw;
     if (isMartian(from)) {
         DHT_LOG.w("Received packet from martian node %s", from.toString().c_str());
         return;
@@ -319,19 +373,37 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr&
         return;
     }
 
-    std::unique_ptr<ParsedMessage> msg {new ParsedMessage};
     try {
-        msgpack::unpacked msg_res = msgpack::unpack((const char*)buf, buflen);
-        msg->msgpack_unpack(msg_res.get());
+        auto unp = msgpack::unpack((const char*)buf, buflen);
+        const auto& o = unp.get();
+        if (o.type == msgpack::type::BIN) {
+            auto decrypted = id_key.decrypt((const uint8_t*)o.via.bin.ptr, o.via.bin.size);
+            msgpack::object_handle oh = msgpack::unpack((const char *)decrypted.data(), decrypted.size());
+            processEncrypted(oh.get(), from);
+        } else {
+            process(o, from);
+        }
     } catch (const std::exception& e) {
         DHT_LOG.w("Can't process message of size %lu: %s", buflen, e.what());
         DHT_LOG.DEBUG.logPrintable(buf, buflen);
         return;
     }
+}
+
+Sp<Node>
+NetworkEngine::process(const msgpack::object& o, const SockAddr& from)
+{
+    std::unique_ptr<ParsedMessage> msg {new ParsedMessage};
+    try {
+        msg->msgpack_unpack(o);
+    } catch (const std::exception& e) {
+        DHT_LOG.w("Can't process message from %s: %s", from.toString().c_str(), e.what());
+        return {};
+    }
 
     if (msg->network != network) {
         DHT_LOG.d("Received message from other network %u", msg->network);
-        return;
+        return {};
     }
 
     const auto& now = scheduler.time();
@@ -342,12 +414,12 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr&
         if (pmsg_it == partial_messages.end()) {
             DHT_LOG.d("Can't find partial message");
             rateLimit(from);
-            return;
+            return {};
         }
         if (!pmsg_it->second.from.equals(from)) {
             DHT_LOG.d("Received partial message data from unexpected IP address");
             rateLimit(from);
-            return;
+            return {};
         }
         // append data block
         if (pmsg_it->second.msg->append(*msg)) {
@@ -358,26 +430,26 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr&
                 process(std::move(pmsg_it->second.msg), from);
                 partial_messages.erase(pmsg_it);
             } else
-                scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, msg->tid));
+                scheduler.add(RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, msg->tid));
         }
-        return;
+        return {};
     }
 
-    if (msg->id == myid || msg->id == zeroes) {
+    if (msg->id == getNodeId() || msg->id == zeroes) {
         DHT_LOG.d("Received message from self");
-        return;
+        return {};
     }
 
     if (msg->type > MessageType::Reply) {
         /* Rate limit requests. */
         if (!rateLimit(from)) {
             DHT_LOG.w("Dropping request due to rate limiting");
-            return;
+            return {};
         }
     }
 
     if (msg->value_parts.empty()) {
-        process(std::move(msg), from);
+        return process(std::move(msg), from);
     } else {
         // starting partial message session
         PartialMessage pmsg;
@@ -387,18 +459,19 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr&
         pmsg.last_part = now;
         auto wmsg = partial_messages.emplace(pmsg.msg->tid, std::move(pmsg));
         if (wmsg.second) {
-            scheduler.add(now + RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, wmsg.first->first));
-            scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, wmsg.first->first));
+            scheduler.add(RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, wmsg.first->first));
+            scheduler.add(RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, wmsg.first->first));
         } else
             DHT_LOG.e("Partial message with given TID already exists");
     }
+    return {};
 }
 
-void
-NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& from)
+Sp<Node>
+NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& from, const Sp<TcpSocket>& tcpsock)
 {
     const auto& now = scheduler.time();
-    auto node = cache.getNode(msg->id, from, now, true, msg->is_client);
+    auto node = cache.getNode(msg->id, from, tcpsock, now, true, msg->is_client);
 
     if (msg->type == MessageType::Error or msg->type == MessageType::Reply) {
         auto rsocket = node->getSocket(msg->tid.toInt());
@@ -415,9 +488,11 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                 node->received(now, req);
                 if (not node->isClient())
                     onNewNode(node, 1);
-                throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find transaction", msg->id};
+                DHT_LOG.e(node->id, "[node %s] Can't find transaction", node->toString().c_str());
+                return node;
             }
         }
+        node->update(from, tcpsock);
 
         node->received(now, req);
 
@@ -427,7 +502,7 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
 
         if (req and (req->cancelled() or req->expired() or req->completed())) {
             DHT_LOG.w(node->id, "[node %s] response to expired, cancelled or completed request", node->toString().c_str());
-            return;
+            return {};
         }
 
         switch (msg->type) {
@@ -473,14 +548,14 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                 ++in_stats.ping;
                 DHT_LOG.d(node->id, "[node %s] sending pong", node->toString().c_str());
                 onPing(node);
-                sendPong(from, msg->tid);
+                sendPong(node, msg->tid);
                 break;
             case MessageType::FindNode: {
                 DHT_LOG.d(msg->target, node->id, "[node %s] got 'find' request for %s (%d)", node->toString().c_str(), msg->target.toString().c_str(), msg->want);
                 ++in_stats.find;
                 RequestAnswer answer = onFindNode(node, msg->target, msg->want);
                 auto nnodes = bufferNodes(from.getFamily(), msg->target, msg->want, answer.nodes4, answer.nodes6);
-                sendNodesValues(from, msg->tid, nnodes.first, nnodes.second, {}, {}, answer.ntoken);
+                sendNodesValues(node, msg->tid, nnodes.first, nnodes.second, {}, {}, answer.ntoken);
                 break;
             }
             case MessageType::GetValues: {
@@ -488,7 +563,7 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                 ++in_stats.get;
                 RequestAnswer answer = onGetValues(node, msg->info_hash, msg->want, msg->query);
                 auto nnodes = bufferNodes(from.getFamily(), msg->info_hash, msg->want, answer.nodes4, answer.nodes6);
-                sendNodesValues(from, msg->tid, nnodes.first, nnodes.second, answer.values, msg->query, answer.ntoken);
+                sendNodesValues(node, msg->tid, nnodes.first, nnodes.second, answer.values, msg->query, answer.ntoken);
                 break;
             }
             case MessageType::AnnounceValue: {
@@ -500,7 +575,7 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                    This is to prevent them from backtracking, and hence
                    polluting the DHT. */
                 for (auto& v : msg->values) {
-                   sendValueAnnounced(from, msg->tid, v->id);
+                   sendValueAnnounced(node, msg->tid, v->id);
                 }
                 break;
             }
@@ -508,13 +583,13 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                 DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'refresh' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                 onRefresh(node, msg->info_hash, msg->token, msg->value_id);
                 /* Same note as above in MessageType::AnnounceValue applies. */
-                sendValueAnnounced(from, msg->tid, msg->value_id);
+                sendValueAnnounced(node, msg->tid, msg->value_id);
                 break;
             case MessageType::Listen: {
                 DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                 ++in_stats.listen;
                 RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id.toInt(), std::move(msg->query));
-                sendListenConfirmation(from, msg->tid);
+                sendListenConfirmation(node, msg->tid);
                 break;
             }
             default:
@@ -523,9 +598,12 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
         } catch (const std::overflow_error& e) {
             DHT_LOG.e("Can't send value: buffer not large enough !");
         } catch (DhtProtocolException& e) {
-            sendError(from, msg->tid, e.getCode(), e.getMsg().c_str(), true);
+            DHT_LOG.w("Error during request handling, sending error ! %p", tcpsock.get());
+            sendError(node, msg->tid, e.getCode(), e.getMsg().c_str(), true, tcpsock);
         }
     }
+
+    return node;
 }
 
 void
@@ -548,38 +626,101 @@ insertAddr(msgpack::packer<msgpack::sbuffer>& pk, const SockAddr& addr)
 }
 
 int
-NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr)
+NetworkEngine::sendUDP(msgpack::sbuffer& msg, const SockAddr& addr)
 {
-    if (not addr)
-        return EFAULT;
+    size_t size = msg.size();
+    uint8_t* data = (uint8_t*)msg.release();
+    return sock->send(data, size, addr);
+}
 
-    int s;
-    if (addr.getFamily() == AF_INET)
-        s = dht_socket;
-    else if (addr.getFamily() == AF_INET6)
-        s = dht_socket6;
-    else
-        s = -1;
+int
+NetworkEngine::send(msgpack::sbuffer& msg, const Node& node, const Sp<TcpSocket>& tcpsock)
+{
+    DHT_LOG.d(node.id, "[node %s] send %lu", node.getId().to_c_str(), msg.size());
+    uint8_t* data;
+    size_t size;
+    if (node.id == InfoHash{}) {
+        size = msg.size();
+        data = (uint8_t*)msg.release();
+    } else {
+        Blob encrypted_out;
+        {
+            Blob encrypted = id_key.encrypt((const uint8_t *)msg.data(), msg.size(), node.id);
+            msgpack::sbuffer buffer;
+            msgpack::packer<msgpack::sbuffer> pk(&buffer);
+            pk.pack_array(2);
+            pk.pack(getNodeId());
+            pk.pack_bin(encrypted.size());
+            pk.pack_bin_body((const char *)encrypted.data(), encrypted.size());
 
-    if (s < 0)
-        return EAFNOSUPPORT;
-    if (sendto(s, buf, len, flags, addr.get(), addr.getLength()) == -1) {
-        int err = errno;
-        DHT_LOG.e("Can't send message to %s: %s", addr.toString().c_str(), strerror(err));
-        return err;
+            encrypted_out = node.id.encrypt((const uint8_t *)buffer.data(), buffer.size());
+        }
+        msgpack::sbuffer buffer;
+        msgpack::packer<msgpack::sbuffer> pk(&buffer);
+        pk.pack_bin(encrypted_out.size());
+        pk.pack_bin_body((const char *)encrypted_out.data(), encrypted_out.size());
+        size = buffer.size();
+        data = (uint8_t*)buffer.release();
+    }
+    if (tcpsock) {
+        return tcpsock->write(data, size);
+    } else if (node.canStream()) {
+        DHT_LOG.d(node.id, "[node %s] send (stream) %lu", node.getId().to_c_str(), msg.size());
+        return node.sock->write(data, size);
+    } else {
+        DHT_LOG.d(node.id, "[node %s] send (datagram) %lu", node.getId().to_c_str(), msg.size());
+        return sock->send(data, size, node.getAddr());
+    }
+}
+
+int
+NetworkEngine::send(const Blob& msg, int /*flags*/, const Node& node, UdpSocket::OnSent&& cb)
+{
+    DHT_LOG.d(node.id, "[node %s] send %lu", node.getId().to_c_str(), msg.size());
+    uint8_t* data;
+    size_t size;
+    if (node.id == InfoHash{}) {
+        size = msg.size();
+        data = (uint8_t*)malloc(size);//(uint8_t*)msg.release();
+        std::memcpy(data, msg.data(), size);
+    } else {
+        Blob encrypted_out;
+        {
+            Blob encrypted = id_key.encrypt((const uint8_t *)msg.data(), msg.size(), node.id);
+            msgpack::sbuffer buffer;
+            msgpack::packer<msgpack::sbuffer> pk(&buffer);
+            pk.pack_array(2);
+            pk.pack(getNodeId());
+            pk.pack_bin(encrypted.size());
+            pk.pack_bin_body((const char *)encrypted.data(), encrypted.size());
+
+            encrypted_out = node.id.encrypt((const uint8_t *)buffer.data(), buffer.size());
+        }
+        msgpack::sbuffer buffer;
+        msgpack::packer<msgpack::sbuffer> pk(&buffer);
+        pk.pack_bin(encrypted_out.size());
+        pk.pack_bin_body((const char *)encrypted_out.data(), encrypted_out.size());
+        size = buffer.size();
+        data = (uint8_t*)buffer.release();
+    }
+    if (node.canStream()) {
+        DHT_LOG.d(node.id, "[node %s] send (stream) %lu", node.getId().to_c_str(), msg.size());
+        return node.sock->write(data, size);
+    } else {
+        DHT_LOG.d(node.id, "[node %s] send (datagram) %lu", node.getId().to_c_str(), msg.size());
+        return sock->send(data, size, node.getAddr(), std::move(cb));
     }
-    return 0;
 }
 
 Sp<Request>
-NetworkEngine::sendPing(Sp<Node> node, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
+NetworkEngine::sendPing(const Sp<Node>& node, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
     auto tid = TransId {TransPrefix::PING, node->getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a")); pk.pack_map(1);
-     pk.pack(std::string("id")); pk.pack(myid);
+     pk.pack(std::string("id")); pk.pack(getNodeId());
 
     pk.pack(std::string("q")); pk.pack(std::string("ping"));
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
@@ -610,14 +751,14 @@ NetworkEngine::sendPing(Sp<Node> node, RequestCb&& on_done, RequestExpiredCb&& o
 }
 
 void
-NetworkEngine::sendPong(const SockAddr& addr, TransId tid) {
+NetworkEngine::sendPong(const Sp<Node>& node, TransId tid) {
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(4+(network?1:0));
 
     pk.pack(std::string("r")); pk.pack_map(2);
-      pk.pack(std::string("id")); pk.pack(myid);
-      insertAddr(pk, addr);
+      pk.pack(std::string("id")); pk.pack(getNodeId());
+      insertAddr(pk, node->getAddress());
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
@@ -627,11 +768,11 @@ NetworkEngine::sendPong(const SockAddr& addr, TransId tid) {
         pk.pack(std::string("n")); pk.pack(network);
     }
 
-    send(buffer.data(), buffer.size(), 0, addr);
+    send(buffer, *node);
 }
 
 Sp<Request>
-NetworkEngine::sendFindNode(Sp<Node> n, const InfoHash& target, want_t want,
+NetworkEngine::sendFindNode(const Sp<Node>& n, const InfoHash& target, want_t want,
         RequestCb&& on_done, RequestExpiredCb&& on_expired) {
     auto tid = TransId {TransPrefix::FIND_NODE, n->getNewTid()};
     msgpack::sbuffer buffer;
@@ -639,7 +780,7 @@ NetworkEngine::sendFindNode(Sp<Node> n, const InfoHash& target, want_t want,
     pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a")); pk.pack_map(2 + (want>0?1:0));
-      pk.pack(std::string("id"));     pk.pack(myid);
+      pk.pack(std::string("id"));     pk.pack(getNodeId());
       pk.pack(std::string("target")); pk.pack(target);
     if (want > 0) {
       pk.pack(std::string("w"));
@@ -677,7 +818,7 @@ NetworkEngine::sendFindNode(Sp<Node> n, const InfoHash& target, want_t want,
 
 
 Sp<Request>
-NetworkEngine::sendGetValues(Sp<Node> n, const InfoHash& info_hash, const Query& query, want_t want,
+NetworkEngine::sendGetValues(const Sp<Node>& n, const InfoHash& info_hash, const Query& query, want_t want,
         RequestCb&& on_done, RequestExpiredCb&& on_expired) {
     auto tid = TransId {TransPrefix::GET_VALUES, n->getNewTid()};
     msgpack::sbuffer buffer;
@@ -687,7 +828,7 @@ NetworkEngine::sendGetValues(Sp<Node> n, const InfoHash& info_hash, const Query&
     pk.pack(std::string("a"));  pk.pack_map(2 +
                                 (query.where.getFilter() or not query.select.getSelection().empty() ? 1:0) +
                                 (want>0?1:0));
-      pk.pack(std::string("id")); pk.pack(myid);
+      pk.pack(std::string("id")); pk.pack(getNodeId());
       pk.pack(std::string("h"));  pk.pack(info_hash);
       pk.pack(std::string("q")); pk.pack(query);
     if (want > 0) {
@@ -751,7 +892,7 @@ NetworkEngine::deserializeNodes(ParsedMessage& msg, const SockAddr& from) {
     for (unsigned i = 0, n = msg.nodes4_raw.size() / NODE4_INFO_BUF_LEN; i < n; i++) {
         const uint8_t* ni = msg.nodes4_raw.data() + i * NODE4_INFO_BUF_LEN;
         const auto& ni_id = *reinterpret_cast<const InfoHash*>(ni);
-        if (ni_id == myid)
+        if (ni_id == getNodeId())
             continue;
         SockAddr addr = deserializeIPv4(ni + ni_id.size());
         if (addr.isLoopback() and from.getFamily() == AF_INET) {
@@ -761,13 +902,13 @@ NetworkEngine::deserializeNodes(ParsedMessage& msg, const SockAddr& from) {
         }
         if (isMartian(addr) || isNodeBlacklisted(addr))
             continue;
-        msg.nodes4.emplace_back(cache.getNode(ni_id, addr, now, false));
+        msg.nodes4.emplace_back(cache.getNode(ni_id, addr, {}, now, false));
         onNewNode(msg.nodes4.back(), 0);
     }
     for (unsigned i = 0, n = msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i < n; i++) {
         const uint8_t* ni = msg.nodes6_raw.data() + i * NODE6_INFO_BUF_LEN;
         const auto& ni_id = *reinterpret_cast<const InfoHash*>(ni);
-        if (ni_id == myid)
+        if (ni_id == getNodeId())
             continue;
         SockAddr addr = deserializeIPv6(ni + ni_id.size());
         if (addr.isLoopback() and from.getFamily() == AF_INET6) {
@@ -777,13 +918,13 @@ NetworkEngine::deserializeNodes(ParsedMessage& msg, const SockAddr& from) {
         }
         if (isMartian(addr) || isNodeBlacklisted(addr))
             continue;
-        msg.nodes6.emplace_back(cache.getNode(ni_id, addr, now, false));
+        msg.nodes6.emplace_back(cache.getNode(ni_id, addr, {}, now, false));
         onNewNode(msg.nodes6.back(), 0);
     }
 }
 
 std::vector<Blob>
-NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, const std::vector<Sp<Value>>& st)
+NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, const std::vector<Sp<Value>>& st, bool stream)
 {
     auto svals = serializeValues(st);
     size_t total_size = 0;
@@ -794,7 +935,7 @@ NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, const std::vector<Sp<Va
     pk.pack(std::string("values"));
     pk.pack_array(svals.size());
     // try to put everything in a single UDP packet
-    if (svals.size() < 50 && total_size < MAX_PACKET_VALUE_SIZE) {
+    if (stream or (svals.size() < 50 and total_size < MAX_PACKET_VALUE_SIZE)) {
         for (const auto& b : svals)
             buffer.write((const char*)b.data(), b.size());
         DHT_LOG.d("sending %lu bytes of values", total_size);
@@ -829,7 +970,7 @@ NetworkEngine::sendValueParts(TransId tid, const std::vector<Blob>& svals, const
                     pk.pack(std::string("o")); pk.pack(start);
                     pk.pack(std::string("d")); pk.pack_bin(end-start);
                                                pk.pack_bin_body((const char*)v.data()+start, end-start);
-            send(buffer.data(), buffer.size(), 0, addr);
+            sendUDP(buffer, addr);
             start = end;
         } while (start != v.size());
         i++;
@@ -837,7 +978,7 @@ NetworkEngine::sendValueParts(TransId tid, const std::vector<Blob>& svals, const
 }
 
 void
-NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& nodes, const Blob& nodes6,
+NetworkEngine::sendNodesValues(const Sp<Node>& node, TransId tid, const Blob& nodes, const Blob& nodes6,
         const std::vector<Sp<Value>>& st, const Query& query, const Blob& token)
 {
     msgpack::sbuffer buffer;
@@ -846,8 +987,8 @@ NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& no
 
     pk.pack(std::string("r"));
     pk.pack_map(2 + (not st.empty()?1:0) + (nodes.size()>0?1:0) + (nodes6.size()>0?1:0) + (not token.empty()?1:0));
-    pk.pack(std::string("id")); pk.pack(myid);
-    insertAddr(pk, addr);
+    pk.pack(std::string("id")); pk.pack(getNodeId());
+    insertAddr(pk, node->getAddress());
     if (nodes.size() > 0) {
         pk.pack(std::string("n4"));
         pk.pack_bin(nodes.size());
@@ -865,7 +1006,7 @@ NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& no
     if (not st.empty()) { /* pack complete values */
         auto fields = query.select.getSelection();
         if (fields.empty()) {
-            svals = packValueHeader(buffer, st);
+            svals = packValueHeader(buffer, st, node->canStream());
         } else { /* pack fields */
             pk.pack(std::string("fields"));
             pk.pack_map(2);
@@ -887,11 +1028,11 @@ NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& no
     }
 
     // send response
-    send(buffer.data(), buffer.size(), 0, addr);
+    send(buffer, *node);
 
     // send parts
     if (not svals.empty())
-        sendValueParts(tid, svals, addr);
+        sendValueParts(tid, svals, node->getAddress());
 }
 
 Blob
@@ -945,7 +1086,7 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, want_t want,
 }
 
 Sp<Request>
-NetworkEngine::sendListen(Sp<Node> n,
+NetworkEngine::sendListen(const Sp<Node>& n,
         const InfoHash& hash,
         const Query& query,
         const Blob& token,
@@ -954,6 +1095,15 @@ NetworkEngine::sendListen(Sp<Node> n,
         RequestExpiredCb&& on_expired,
         SocketCb&& socket_cb)
 {
+    //n->startTcp(scheduler.getLoop());
+    if (not n->sock) {
+        n->sock = std::make_shared<TcpSocket>(scheduler.getLoop());
+        n->sock->connect(n->getAddr().get(), [this,n](int status){
+            if (status == 0)
+                startTcp(n->sock, n);
+        });
+    }
+
     uint32_t socket;
     auto tid = TransId { TransPrefix::LISTEN, n->getNewTid() };
     if (previous and previous->node == n) {
@@ -976,7 +1126,7 @@ NetworkEngine::sendListen(Sp<Node> n,
 
     auto has_query = query.where.getFilter() or not query.select.getSelection().empty();
     pk.pack(std::string("a")); pk.pack_map(4 + has_query);
-      pk.pack(std::string("id"));    pk.pack(myid);
+      pk.pack(std::string("id"));    pk.pack(getNodeId());
       pk.pack(std::string("h"));     pk.pack(hash);
       pk.pack(std::string("token")); packToken(pk, token);
       pk.pack(std::string("sid"));  pk.pack_bin(sid.size());
@@ -1012,14 +1162,14 @@ NetworkEngine::sendListen(Sp<Node> n,
 }
 
 void
-NetworkEngine::sendListenConfirmation(const SockAddr& addr, TransId tid) {
+NetworkEngine::sendListenConfirmation(const Sp<Node>& node, TransId tid) {
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(4+(network?1:0));
 
     pk.pack(std::string("r")); pk.pack_map(2);
-      pk.pack(std::string("id")); pk.pack(myid);
-      insertAddr(pk, addr);
+      pk.pack(std::string("id")); pk.pack(getNodeId());
+      insertAddr(pk, node->getAddress());
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
@@ -1029,11 +1179,11 @@ NetworkEngine::sendListenConfirmation(const SockAddr& addr, TransId tid) {
         pk.pack(std::string("n")); pk.pack(network);
     }
 
-    send(buffer.data(), buffer.size(), 0, addr);
+    send(buffer, *node);
 }
 
 Sp<Request>
-NetworkEngine::sendAnnounceValue(Sp<Node> n,
+NetworkEngine::sendAnnounceValue(const Sp<Node>& n,
         const InfoHash& infohash,
         const Sp<Value>& value,
         time_point created,
@@ -1047,9 +1197,9 @@ NetworkEngine::sendAnnounceValue(Sp<Node> n,
     pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a")); pk.pack_map((created < scheduler.time() ? 5 : 4));
-      pk.pack(std::string("id"));     pk.pack(myid);
+      pk.pack(std::string("id"));     pk.pack(getNodeId());
       pk.pack(std::string("h"));      pk.pack(infohash);
-      auto v = packValueHeader(buffer, {value});
+      auto v = packValueHeader(buffer, {value}, n->canStream());
       if (created < scheduler.time()) {
           pk.pack(std::string("c"));
           pk.pack(to_time_t(created));
@@ -1092,7 +1242,7 @@ NetworkEngine::sendAnnounceValue(Sp<Node> n,
 }
 
 Sp<Request>
-NetworkEngine::sendRefreshValue(Sp<Node> n,
+NetworkEngine::sendRefreshValue(const Sp<Node>& n,
                 const InfoHash& infohash,
                 const Value::Id& vid,
                 const Blob& token,
@@ -1105,7 +1255,7 @@ NetworkEngine::sendRefreshValue(Sp<Node> n,
     pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a")); pk.pack_map(4);
-      pk.pack(std::string("id"));  pk.pack(myid);
+      pk.pack(std::string("id"));  pk.pack(getNodeId());
       pk.pack(std::string("h"));  pk.pack(infohash);
       pk.pack(std::string("vid")); pk.pack(vid);
       pk.pack(std::string("token"));  pk.pack(token);
@@ -1144,15 +1294,15 @@ NetworkEngine::sendRefreshValue(Sp<Node> n,
 }
 
 void
-NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id vid) {
+NetworkEngine::sendValueAnnounced(const Sp<Node>& node, TransId tid, Value::Id vid) {
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_map(4+(network?1:0));
 
     pk.pack(std::string("r")); pk.pack_map(3);
-      pk.pack(std::string("id"));  pk.pack(myid);
+      pk.pack(std::string("id"));  pk.pack(getNodeId());
       pk.pack(std::string("vid")); pk.pack(vid);
-      insertAddr(pk, addr);
+      insertAddr(pk, node->getAddress());
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
@@ -1162,15 +1312,16 @@ NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id v
         pk.pack(std::string("n")); pk.pack(network);
     }
 
-    send(buffer.data(), buffer.size(), 0, addr);
+    send(buffer, *node);
 }
 
 void
-NetworkEngine::sendError(const SockAddr& addr,
+NetworkEngine::sendError(const Sp<Node>& node,
         TransId tid,
         uint16_t code,
         const std::string& message,
-        bool include_id)
+        bool include_id,
+        const Sp<TcpSocket>& tcpsock)
 {
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
@@ -1182,7 +1333,7 @@ NetworkEngine::sendError(const SockAddr& addr,
 
     if (include_id) {
         pk.pack(std::string("r")); pk.pack_map(1);
-          pk.pack(std::string("id")); pk.pack(myid);
+          pk.pack(std::string("id")); pk.pack(getNodeId());
     }
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
@@ -1193,7 +1344,7 @@ NetworkEngine::sendError(const SockAddr& addr,
         pk.pack(std::string("n")); pk.pack(network);
     }
 
-    send(buffer.data(), buffer.size(), 0, addr);
+    send(buffer, *node, tcpsock);
 }
 
 void

src/node.cpp

@@ -31,8 +31,8 @@ constexpr std::chrono::minutes Node::NODE_GOOD_TIME;
 constexpr std::chrono::seconds Node::MAX_RESPONSE_TIME;
 
 
-Node::Node(const InfoHash& id, const SockAddr& addr, bool client)
-: id(id), addr(addr), is_client(client), sockets_()
+Node::Node(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& s, bool client)
+: id(id), addr(addr), sock(s), is_client(client), sockets_()
 {
     crypto::random_device rd;
     transaction_id = std::uniform_int_distribution<SocketId>{1}(rd);
@@ -69,9 +69,11 @@ Node::getPendingMessageCount() const
 }
 
 void
-Node::update(const SockAddr& new_addr)
+Node::update(const SockAddr& new_addr, const Sp<TcpSocket>& s)
 {
     addr = new_addr;
+    if (not sock or (s and sock->isClosed() and not s->isClosed()))
+        sock = s;
 }
 
 /** To be called when a message was sent to the node */

src/node_cache.cpp

@@ -26,10 +26,10 @@ NodeCache::getNode(const InfoHash& id, sa_family_t family) {
 }
 
 Sp<Node>
-NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) {
+NodeCache::getNode(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& sock, time_point now, bool confirm, bool client) {
     if (id == zeroes)
         return std::make_shared<Node>(id, addr);
-    return cache(addr.getFamily()).getNode(id, addr, now, confirm, client);
+    return cache(addr.getFamily()).getNode(id, addr, sock, now, confirm, client);
 }
 
 std::vector<Sp<Node>>
@@ -78,6 +78,17 @@ NodeCache::clearBadNodes(sa_family_t family)
     }
 }
 
+void
+NodeCache::closeAll(sa_family_t family)
+{
+    if (family == 0) {
+        closeAll(AF_INET);
+        closeAll(AF_INET6);
+    } else {
+        (family == AF_INET ? cache_4 : cache_6).closeAll();
+    }
+}
+
 Sp<Node>
 NodeCache::NodeMap::getNode(const InfoHash& id)
 {
@@ -91,15 +102,15 @@ NodeCache::NodeMap::getNode(const InfoHash& id)
 }
 
 Sp<Node>
-NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client)
+NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, const Sp<TcpSocket>& sock, time_point now, bool confirm, bool client)
 {
     auto it = emplace(id, std::weak_ptr<Node>{});
     auto node = it.first->second.lock();
     if (not node) {
-        node = std::make_shared<Node>(id, addr, client);
+        node = std::make_shared<Node>(id, addr, sock, client);
         it.first->second = node;
     } else if (confirm or node->isOld(now)) {
-        node->update(addr);
+        node->update(addr, sock);
     }
     return node;
 }
@@ -116,4 +127,21 @@ NodeCache::NodeMap::clearBadNodes() {
     }
 }
 
+void
+NodeCache::NodeMap::closeAll()
+{
+    for (auto it = cbegin(); it != cend();) {
+        if (auto n = it->second.lock()) {
+            if (n->sock) {
+                n->sock->close();
+                n->sock.reset();
+            }
+            ++it;
+        } else {
+            erase(it++);
+        }
+    }
+}
+
+
 }

src/search.h

@@ -320,7 +320,7 @@ struct Dht::Search {
     uint16_t tid;
     time_point refill_time {time_point::min()};
     time_point step_time {time_point::min()};           /* the time of the last search step */
-    Sp<Scheduler::Job> nextSearchStep {};
+    Sp<Job> nextSearchStep {};
 
     bool expired {false};              /* no node, or all nodes expired */
     bool done {false};                 /* search is over, cached for later */

src/securedht.cpp

@@ -40,10 +40,10 @@ Config& getConfig(SecureDht::Config& conf)
     return c;
 }
 
-SecureDht::SecureDht(int s, int s6, SecureDht::Config conf)
-: Dht(s, s6, getConfig(conf)), key_(conf.id.first), certificate_(conf.id.second)
+SecureDht::SecureDht(uv_loop_t* loop, SecureDht::Config conf)
+: Dht(loop, getConfig(conf)), key_(conf.id.first), certificate_(conf.id.second)
 {
-    if (s < 0 && s6 < 0)
+    if (not loop)
         return;
 
     for (const auto& type : DEFAULT_TYPES)

src/utils.cpp

@@ -20,12 +20,15 @@
 #include "sockaddr.h"
 #include "default_types.h"
 #include "net.h"
+#include "uv_utils.h"
 
 /* An IPv4 equivalent to IN6_IS_ADDR_UNSPECIFIED */
 #ifndef IN_IS_ADDR_UNSPECIFIED
 #define IN_IS_ADDR_UNSPECIFIED(a) (((long int) (a)->s_addr) == 0x00000000)
 #endif /* IN_IS_ADDR_UNSPECIFIED */
 
+static constexpr std::array<uint8_t, 12> MAPPED_IPV4_PREFIX {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}};
+
 namespace dht {
 
 std::string
@@ -112,6 +115,57 @@ SockAddr::isPrivate() const
     }
 }
 
+bool
+SockAddr::isMappedIPv4() const
+{
+    if (getFamily() != AF_INET6)
+        return false;
+    const uint8_t* addr6 = reinterpret_cast<const uint8_t*>(&getIPv6().sin6_addr);
+    return std::equal(MAPPED_IPV4_PREFIX.begin(), MAPPED_IPV4_PREFIX.end(), addr6);
+}
+
+SockAddr
+SockAddr::getMappedIPv4() const
+{
+    if (getFamily() != AF_INET6)
+        return *this;
+    SockAddr ret;
+    ret.setFamily(AF_INET);
+    ret.setPort(getPort());
+    auto addr6 = reinterpret_cast<const uint8_t*>(&getIPv6().sin6_addr);
+    auto addr4 = reinterpret_cast<uint8_t*>(&ret.getIPv4().sin_addr);
+    addr6 += MAPPED_IPV4_PREFIX.size();
+    std::copy_n(addr6, sizeof(in_addr), addr4);
+    return ret;
+}
+
+void get_addr_info_cb(uv_getaddrinfo_t* req, int status, struct addrinfo* res) {
+    if (auto cb = static_cast<GetAddrInfoCb*>(req->data)) {
+        size_t count {0};
+        for (auto addr = res; addr; addr = addr->ai_next)
+            count++;
+        std::vector<SockAddr> ret;
+        ret.reserve(count);
+        for (auto addr = res; addr; addr = addr->ai_next)
+            ret.emplace_back(addr->ai_addr, addr->ai_addrlen);
+        (*cb)(std::move(ret));
+        delete cb;
+    }
+    delete req;
+    uv_freeaddrinfo(res);
+}
+
+void getAddrInfo(uv_loop_t* loop, const char* node, const char* service, GetAddrInfoCb&& cb)
+{
+    auto req = new uv_getaddrinfo_t;
+    req->data = new GetAddrInfoCb(std::move(cb));
+    addrinfo hints {};
+    hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG;
+    hints.ai_family = AF_UNSPEC;
+    hints.ai_socktype = SOCK_DGRAM;
+    uv_getaddrinfo(loop, req, get_addr_info_cb, node, service, &hints);
+}
+
 bool operator==(const SockAddr& a, const SockAddr& b) {
     return a.equals(b);
 }

tools/dhtnode.cpp

@@ -39,7 +39,7 @@ void print_id_req() {
 }
 
 void print_node_info(const std::shared_ptr<DhtRunner>& dht, const dht_params& params) {
-    std::cout << "OpenDht node " << dht->getNodeId() << " running on port " <<  dht->getBoundPort() << std::endl;
+    std::cout << "OpenDht node " << dht->getNodeId() << " running on port " <<  dht->getBound().toString() << std::endl;
     if (params.generate_identity)
         std::cout << "Public key ID " << dht->getId() << std::endl;
 }
@@ -370,6 +370,9 @@ main(int argc, char **argv)
         }
 
         dht->run(params.port, crt, true, params.network);
+        dht->setOnStatusChanged([](dht::NodeStatus status4, dht::NodeStatus status6){
+            std::cout << "Node status changed : IPv4:" << (unsigned)status4 << " IPv6:" << (unsigned)status6 << std::endl;
+        });
 
         if (params.log) {
             if (params.syslog or (params.daemonize and params.logfile.empty()))